diff options
Diffstat (limited to 'arch')
888 files changed, 10178 insertions, 20685 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 678a80713b21..9af3a597cc67 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -24,9 +24,6 @@ config KEXEC_ELF config HAVE_IMA_KEXEC bool -config SET_FS - bool - config HOTPLUG_SMT bool @@ -599,21 +596,22 @@ config STACKPROTECTOR_STRONG config ARCH_SUPPORTS_SHADOW_CALL_STACK bool help - An architecture should select this if it supports Clang's Shadow - Call Stack and implements runtime support for shadow stack + An architecture should select this if it supports the compiler's + Shadow Call Stack and implements runtime support for shadow stack switching. config SHADOW_CALL_STACK - bool "Clang Shadow Call Stack" - depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + bool "Shadow Call Stack" + depends on ARCH_SUPPORTS_SHADOW_CALL_STACK depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER help - This option enables Clang's Shadow Call Stack, which uses a - shadow stack to protect function return addresses from being - overwritten by an attacker. More information can be found in - Clang's documentation: + This option enables the compiler's Shadow Call Stack, which + uses a shadow stack to protect function return addresses from + being overwritten by an attacker. More information can be found + in the compiler's documentation: - https://clang.llvm.org/docs/ShadowCallStack.html + - Clang: https://clang.llvm.org/docs/ShadowCallStack.html + - GCC: https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#Instrumentation-Options Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses @@ -898,6 +896,13 @@ config HAVE_SOFTIRQ_ON_OWN_STACK Architecture provides a function to run __do_softirq() on a separate stack. +config ALTERNATE_USER_ADDRESS_SPACE + bool + help + Architectures set this when the CPU uses separate address + spaces for kernel and user space pointers. In this case, the + access_ok() check on a __user pointer is skipped. + config PGTABLE_LEVELS int default 2 @@ -1159,16 +1164,30 @@ config HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET to the compiler, so it will attempt to add canary checks regardless of the static branch state. -config RANDOMIZE_KSTACK_OFFSET_DEFAULT - bool "Randomize kernel stack offset on syscall entry" +config RANDOMIZE_KSTACK_OFFSET + bool "Support for randomizing kernel stack offset on syscall entry" if EXPERT + default y depends on HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET + depends on INIT_STACK_NONE || !CC_IS_CLANG || CLANG_VERSION >= 140000 help The kernel stack offset can be randomized (after pt_regs) by roughly 5 bits of entropy, frustrating memory corruption attacks that depend on stack address determinism or - cross-syscall address exposures. This feature is controlled - by kernel boot param "randomize_kstack_offset=on/off", and this - config chooses the default boot state. + cross-syscall address exposures. + + The feature is controlled via the "randomize_kstack_offset=on/off" + kernel boot param, and if turned off has zero overhead due to its use + of static branches (see JUMP_LABEL). + + If unsure, say Y. + +config RANDOMIZE_KSTACK_OFFSET_DEFAULT + bool "Default state of kernel stack offset randomization" + depends on RANDOMIZE_KSTACK_OFFSET + help + Kernel stack offset randomization is controlled by kernel boot param + "randomize_kstack_offset=on/off", and this config chooses the default + boot state. config ARCH_OPTIONAL_KERNEL_RWX def_bool n @@ -1278,12 +1297,41 @@ config HAVE_STATIC_CALL_INLINE config HAVE_PREEMPT_DYNAMIC bool + +config HAVE_PREEMPT_DYNAMIC_CALL + bool depends on HAVE_STATIC_CALL - depends on GENERIC_ENTRY + select HAVE_PREEMPT_DYNAMIC help - Select this if the architecture support boot time preempt setting - on top of static calls. It is strongly advised to support inline - static call to avoid any overhead. + An architecture should select this if it can handle the preemption + model being selected at boot time using static calls. + + Where an architecture selects HAVE_STATIC_CALL_INLINE, any call to a + preemption function will be patched directly. + + Where an architecture does not select HAVE_STATIC_CALL_INLINE, any + call to a preemption function will go through a trampoline, and the + trampoline will be patched. + + It is strongly advised to support inline static call to avoid any + overhead. + +config HAVE_PREEMPT_DYNAMIC_KEY + bool + depends on HAVE_ARCH_JUMP_LABEL && CC_HAS_ASM_GOTO + select HAVE_PREEMPT_DYNAMIC + help + An architecture should select this if it can handle the preemption + model being selected at boot time using static keys. + + Each preemption function will be given an early return based on a + static key. This should have slightly lower overhead than non-inline + static calls, as this effectively inlines each trampoline into the + start of its callee. This may avoid redundant work, and may + integrate better with CFI schemes. + + This will have greater overhead than using inline static calls as + the call to the preemption function cannot be entirely elided. config ARCH_WANT_LD_ORPHAN_WARN bool diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 4e87783c90ad..7d0d26b5b3f5 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -12,7 +12,6 @@ config ALPHA select FORCE_PCI if !ALPHA_JENSEN select PCI_DOMAINS if PCI select PCI_SYSCALL if PCI - select HAVE_AOUT select HAVE_ASM_MODVERSIONS select HAVE_PCSPKR_PLATFORM select HAVE_PERF_EVENTS @@ -35,7 +34,6 @@ config ALPHA select OLD_SIGSUSPEND select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 select MMU_GATHER_NO_RANGE - select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM select ZONE_DMA help diff --git a/arch/alpha/configs/defconfig b/arch/alpha/configs/defconfig index 7f1ca30b115b..7e9336930880 100644 --- a/arch/alpha/configs/defconfig +++ b/arch/alpha/configs/defconfig @@ -62,7 +62,6 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=m CONFIG_NFS_V3=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 02f0429f1068..170451fde043 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -233,6 +233,7 @@ pmd_page_vaddr(pmd_t pmd) return ((pmd_val(pmd) & _PFN_MASK) >> (32-PAGE_SHIFT)) + PAGE_OFFSET; } +#define pmd_pfn(pmd) (pmd_val(pmd) >> 32) #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> 32)) #define pud_page(pud) (pfn_to_page(pud_val(pud) >> 32)) diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h index 090499c99c1c..43e234c518b1 100644 --- a/arch/alpha/include/asm/processor.h +++ b/arch/alpha/include/asm/processor.h @@ -26,10 +26,6 @@ #define TASK_UNMAPPED_BASE \ ((current->personality & ADDR_LIMIT_32BIT) ? 0x40000000 : TASK_SIZE / 2) -typedef struct { - unsigned long seg; -} mm_segment_t; - /* This is dead. Everything has been moved to thread_info. */ struct thread_struct { }; #define INIT_THREAD { } diff --git a/arch/alpha/include/asm/thread_info.h b/arch/alpha/include/asm/thread_info.h index 2592356e3215..fdc485d7787a 100644 --- a/arch/alpha/include/asm/thread_info.h +++ b/arch/alpha/include/asm/thread_info.h @@ -19,7 +19,6 @@ struct thread_info { unsigned int flags; /* low level flags */ unsigned int ieee_state; /* see fpu.h */ - mm_segment_t addr_limit; /* thread address space */ unsigned cpu; /* current CPU */ int preempt_count; /* 0 => preemptable, <0 => BUG */ unsigned int status; /* thread-synchronous flags */ @@ -35,7 +34,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .addr_limit = KERNEL_DS, \ .preempt_count = INIT_PREEMPT_COUNT, \ } diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 1b6f25efa247..c32c2584c0b7 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -2,47 +2,7 @@ #ifndef __ALPHA_UACCESS_H #define __ALPHA_UACCESS_H -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * Or at least it did once upon a time. Nowadays it is a mask that - * defines which bits of the address space are off limits. This is a - * wee bit faster than the above. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define KERNEL_DS ((mm_segment_t) { 0UL }) -#define USER_DS ((mm_segment_t) { -0x40000000000UL }) - -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -/* - * Is a address valid? This does a straightforward calculation rather - * than tests. - * - * Address valid if: - * - "addr" doesn't have any high-bits set - * - AND "size" doesn't have any high-bits set - * - AND "addr+size-(size != 0)" doesn't have any high-bits set - * - OR we are in kernel mode. - */ -#define __access_ok(addr, size) ({ \ - unsigned long __ao_a = (addr), __ao_b = (size); \ - unsigned long __ao_end = __ao_a + __ao_b - !!__ao_b; \ - (get_fs().seg & (__ao_a | __ao_b | __ao_end)) == 0; }) - -#define access_ok(addr, size) \ -({ \ - __chk_user_ptr(addr); \ - __access_ok(((unsigned long)(addr)), (size)); \ -}) - +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. @@ -105,7 +65,7 @@ extern void __get_user_unknown(void); long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - if (__access_ok((unsigned long)__gu_addr, size)) { \ + if (__access_ok(__gu_addr, size)) { \ __gu_err = 0; \ switch (size) { \ case 1: __get_user_8(__gu_addr); break; \ @@ -200,7 +160,7 @@ extern void __put_user_unknown(void); ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - if (__access_ok((unsigned long)__pu_addr, size)) { \ + if (__access_ok(__pu_addr, size)) { \ __pu_err = 0; \ switch (size) { \ case 1: __put_user_8(x, __pu_addr); break; \ @@ -316,17 +276,14 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long len) extern long __clear_user(void __user *to, long len); -extern inline long +static inline long clear_user(void __user *to, long len) { - if (__access_ok((unsigned long)to, len)) + if (__access_ok(to, len)) len = __clear_user(to, len); return len; } -#define user_addr_max() \ - (uaccess_kernel() ? ~0UL : TASK_SIZE) - extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strnlen_user(const char __user *str, long n); diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h index 5aeb4fb3cb7c..e0de0c233ab9 100644 --- a/arch/alpha/include/asm/xor.h +++ b/arch/alpha/include/asm/xor.h @@ -5,24 +5,43 @@ * Optimized RAID-5 checksumming functions for alpha EV5 and EV6 */ -extern void xor_alpha_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_alpha_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_alpha_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_alpha_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +extern void +xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +extern void +xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +extern void +xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +extern void +xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); -extern void xor_alpha_prefetch_2(unsigned long, unsigned long *, - unsigned long *); -extern void xor_alpha_prefetch_3(unsigned long, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_alpha_prefetch_4(unsigned long, unsigned long *, - unsigned long *, unsigned long *, - unsigned long *); -extern void xor_alpha_prefetch_5(unsigned long, unsigned long *, - unsigned long *, unsigned long *, - unsigned long *, unsigned long *); +extern void +xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +extern void +xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +extern void +xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +extern void +xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); asm(" \n\ .text \n\ diff --git a/arch/alpha/include/uapi/asm/signal.h b/arch/alpha/include/uapi/asm/signal.h index a69dd8d080a8..1413075f7616 100644 --- a/arch/alpha/include/uapi/asm/signal.h +++ b/arch/alpha/include/uapi/asm/signal.h @@ -100,7 +100,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; /* sigstack(2) is deprecated, and will be withdrawn in a future version diff --git a/arch/alpha/mm/init.c b/arch/alpha/mm/init.c index f6114d03357c..7511723b7669 100644 --- a/arch/alpha/mm/init.c +++ b/arch/alpha/mm/init.c @@ -76,14 +76,14 @@ pgd_alloc(struct mm_struct *mm) pmd_t * __bad_pagetable(void) { - memset((void *) EMPTY_PGT, 0, PAGE_SIZE); + memset(absolute_pointer(EMPTY_PGT), 0, PAGE_SIZE); return (pmd_t *) EMPTY_PGT; } pte_t __bad_page(void) { - memset((void *) EMPTY_PGE, 0, PAGE_SIZE); + memset(absolute_pointer(EMPTY_PGE), 0, PAGE_SIZE); return pte_mkdirty(mk_pte(virt_to_page(EMPTY_PGE), PAGE_SHARED)); } @@ -253,7 +253,7 @@ void __init paging_init(void) free_area_init(max_zone_pfn); /* Initialize the kernel's ZERO_PGE. */ - memset((void *)ZERO_PGE, 0, PAGE_SIZE); + memset(absolute_pointer(ZERO_PGE), 0, PAGE_SIZE); } #if defined(CONFIG_ALPHA_GENERIC) || defined(CONFIG_ALPHA_SRM) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 3c2a4753d09b..e0a60a27e14d 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -45,7 +45,6 @@ config ARC select PCI_SYSCALL if PCI select PERF_USE_VMALLOC if ARC_CACHE_VIPT_ALIASING select HAVE_ARCH_JUMP_LABEL if ISA_ARCV2 && !CPU_ENDIAN_BE32 - select SET_FS select TRACE_IRQFLAGS_SUPPORT config LOCKDEP_SUPPORT diff --git a/arch/arc/include/asm/hugepage.h b/arch/arc/include/asm/hugepage.h index 11b0ff26b97b..5001b796fb8d 100644 --- a/arch/arc/include/asm/hugepage.h +++ b/arch/arc/include/asm/hugepage.h @@ -31,7 +31,6 @@ static inline pmd_t pte_pmd(pte_t pte) #define pmd_write(pmd) pte_write(pmd_pte(pmd)) #define pmd_young(pmd) pte_young(pmd_pte(pmd)) -#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) #define mk_pmd(page, prot) pte_pmd(mk_pte(page, prot)) diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h index 8084ef2f6491..7848348719b2 100644 --- a/arch/arc/include/asm/pgtable-levels.h +++ b/arch/arc/include/asm/pgtable-levels.h @@ -161,6 +161,7 @@ #define pmd_present(x) (pmd_val(x)) #define pmd_clear(xp) do { pmd_val(*(xp)) = 0; } while (0) #define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK) +#define pmd_pfn(pmd) ((pmd_val(pmd) & PAGE_MASK) >> PAGE_SHIFT) #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd)) #define set_pmd(pmdp, pmd) (*(pmdp) = pmd) #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) diff --git a/arch/arc/include/asm/segment.h b/arch/arc/include/asm/segment.h deleted file mode 100644 index 871f8ab11bfd..000000000000 --- a/arch/arc/include/asm/segment.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) - */ - -#ifndef __ASMARC_SEGMENT_H -#define __ASMARC_SEGMENT_H - -#ifndef __ASSEMBLY__ - -typedef unsigned long mm_segment_t; - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#define KERNEL_DS MAKE_MM_SEG(0) -#define USER_DS MAKE_MM_SEG(TASK_SIZE) -#define uaccess_kernel() (get_fs() == KERNEL_DS) - -#endif /* __ASSEMBLY__ */ -#endif /* __ASMARC_SEGMENT_H */ diff --git a/arch/arc/include/asm/thread_info.h b/arch/arc/include/asm/thread_info.h index d36863e34bfc..1e0b2e3914d5 100644 --- a/arch/arc/include/asm/thread_info.h +++ b/arch/arc/include/asm/thread_info.h @@ -27,7 +27,6 @@ #ifndef __ASSEMBLY__ #include <linux/thread_info.h> -#include <asm/segment.h> /* * low level task data that entry.S needs immediate access to @@ -40,7 +39,6 @@ struct thread_info { unsigned long flags; /* low level flags */ int preempt_count; /* 0 => preemptable, <0 => BUG */ struct task_struct *task; /* main task structure */ - mm_segment_t addr_limit; /* thread address space */ __u32 cpu; /* current CPU */ unsigned long thr_ptr; /* TLS ptr */ }; @@ -56,7 +54,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } static inline __attribute_const__ struct thread_info *current_thread_info(void) diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 783bfdb3bfa3..99712471c96a 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -23,35 +23,6 @@ #include <linux/string.h> /* for generic string functions */ - -#define __kernel_ok (uaccess_kernel()) - -/* - * Algorithmically, for __user_ok() we want do: - * (start < TASK_SIZE) && (start+len < TASK_SIZE) - * where TASK_SIZE could either be retrieved from thread_info->addr_limit or - * emitted directly in code. - * - * This can however be rewritten as follows: - * (len <= TASK_SIZE) && (start+len < TASK_SIZE) - * - * Because it essentially checks if buffer end is within limit and @len is - * non-ngeative, which implies that buffer start will be within limit too. - * - * The reason for rewriting being, for majority of cases, @len is generally - * compile time constant, causing first sub-expression to be compile time - * subsumed. - * - * The second part would generate weird large LIMMs e.g. (0x6000_0000 - 0x10), - * so we check for TASK_SIZE using get_fs() since the addr_limit load from mem - * would already have been done at this call site for __kernel_ok() - * - */ -#define __user_ok(addr, sz) (((sz) <= TASK_SIZE) && \ - ((addr) <= (get_fs() - (sz)))) -#define __access_ok(addr, sz) (unlikely(__kernel_ok) || \ - likely(__user_ok((addr), (sz)))) - /*********** Single byte/hword/word copies ******************/ #define __get_user_fn(sz, u, k) \ @@ -667,7 +638,6 @@ extern unsigned long arc_clear_user_noinline(void __user *to, #define __clear_user(d, n) arc_clear_user_noinline(d, n) #endif -#include <asm/segment.h> #include <asm-generic/uaccess.h> #endif diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 8e90052f6f05..5f7f5aab361f 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -43,7 +43,7 @@ SYSCALL_DEFINE0(arc_gettls) return task_thread_info(current)->thr_ptr; } -SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) +SYSCALL_DEFINE3(arc_usr_cmpxchg, int __user *, uaddr, int, expected, int, new) { struct pt_regs *regs = current_pt_regs(); u32 uval; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4c97cb40eebb..c0d14b192b27 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -5,6 +5,7 @@ config ARM select ARCH_32BIT_OFF_T select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE if HAVE_KRETPROBES && FRAME_POINTER && !ARM_UNWIND select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE select ARCH_HAS_ELF_RANDOMIZE @@ -37,6 +38,7 @@ config ARM select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_MEMTEST select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU + select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WANT_LD_ORPHAN_WARN select BINFMT_FLAT_ARGVP_ENVP_ON_STACK @@ -58,6 +60,7 @@ config ARM select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL @@ -92,8 +95,8 @@ config ARM select HAVE_EXIT_THREAD select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL - select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG) + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER if !XIP_KERNEL select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT if PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7) select HAVE_IRQ_TIME_ACCOUNTING @@ -127,7 +130,8 @@ config ARM select PERF_USE_VMALLOC select RTC_LIB select SYS_SUPPORTS_APM_EMULATION - select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO + select THREAD_INFO_IN_TASK + select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. @@ -139,6 +143,17 @@ config ARM Europe. There is an ARM Linux project with a web page at <http://www.arm.linux.org.uk/>. +config ARM_HAS_GROUP_RELOCS + def_bool y + depends on !LD_IS_LLD || LLD_VERSION >= 140000 + depends on !COMPILE_TEST + help + Whether or not to use R_ARM_ALU_PC_Gn or R_ARM_LDR_PC_Gn group + relocations, which have been around for a long time, but were not + supported in LLD until version 14. The combined range is -/+ 256 MiB, + which is usually sufficient, but not for allyesconfig, so we disable + this feature when doing compile testing. + config ARM_HAS_SG_CHAIN bool @@ -227,9 +242,6 @@ config GENERIC_ISA_DMA config FIQ bool -config NEED_RET_TO_USER - bool - config ARCH_MTD_XIP bool @@ -323,7 +335,6 @@ config ARCH_MULTIPLATFORM select AUTO_ZRELADDR select TIMER_OF select COMMON_CLK - select GENERIC_IRQ_MULTI_HANDLER select HAVE_PCI select PCI_DOMAINS_GENERIC if PCI select SPARSE_IRQ @@ -347,7 +358,6 @@ config ARCH_EP93XX select ARM_AMBA imply ARM_PATCH_PHYS_VIRT select ARM_VIC - select GENERIC_IRQ_MULTI_HANDLER select AUTO_ZRELADDR select CLKSRC_MMIO select CPU_ARM920T @@ -372,7 +382,6 @@ config ARCH_IOP32X select CPU_XSCALE select GPIO_IOP select GPIOLIB - select NEED_RET_TO_USER select FORCE_PCI select PLAT_IOP help @@ -386,7 +395,6 @@ config ARCH_IXP4XX select ARCH_SUPPORTS_BIG_ENDIAN select CPU_XSCALE select DMABOUNCE if PCI - select GENERIC_IRQ_MULTI_HANDLER select GPIO_IXP4XX select GPIOLIB select HAVE_PCI @@ -402,7 +410,6 @@ config ARCH_IXP4XX config ARCH_DOVE bool "Marvell Dove" select CPU_PJ4 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_PCI select MVEBU_MBUS @@ -425,7 +432,6 @@ config ARCH_PXA select CLKSRC_MMIO select TIMER_OF select CPU_XSCALE if !CPU_XSC3 - select GENERIC_IRQ_MULTI_HANDLER select GPIO_PXA select GPIOLIB select IRQ_DOMAIN @@ -464,7 +470,6 @@ config ARCH_SA1100 select COMMON_CLK select CPU_FREQ select CPU_SA1100 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select IRQ_DOMAIN select ISA @@ -479,7 +484,6 @@ config ARCH_S3C24XX select CLKSRC_SAMSUNG_PWM select GPIO_SAMSUNG select GPIOLIB - select GENERIC_IRQ_MULTI_HANDLER select NEED_MACH_IO_H select S3C2410_WATCHDOG select SAMSUNG_ATAGS @@ -497,7 +501,6 @@ config ARCH_OMAP1 select ARCH_OMAP select CLKSRC_MMIO select GENERIC_IRQ_CHIP - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_LEGACY_CLK select IRQ_DOMAIN @@ -1164,7 +1167,12 @@ config SMP_ON_UP config CURRENT_POINTER_IN_TPIDRURO def_bool y - depends on SMP && CPU_32v6K && !CPU_V6 + depends on CPU_32v6K && !CPU_V6 + +config IRQSTACKS + def_bool y + select HAVE_IRQ_EXIT_ON_IRQ_STACK + select HAVE_SOFTIRQ_ON_OWN_STACK config ARM_CPU_TOPOLOGY bool "Support cpu topology definition" @@ -1508,9 +1516,6 @@ config HW_PERF_EVENTS def_bool y depends on ARM_PMU -config ARCH_WANT_GENERAL_HUGETLB - def_bool y - config ARM_MODULE_PLTS bool "Use PLTs to allow module memory to spill over into vmalloc area" depends on MODULES @@ -1608,10 +1613,14 @@ config XEN help Say Y if you want to run Linux in a Virtual Machine on Xen on ARM. +config CC_HAVE_STACKPROTECTOR_TLS + def_bool $(cc-option,-mtp=cp15 -mstack-protector-guard=tls -mstack-protector-guard-offset=0) + config STACKPROTECTOR_PER_TASK bool "Use a unique stack canary value for each task" - depends on GCC_PLUGINS && STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA - select GCC_PLUGIN_ARM_SSP_PER_TASK + depends on STACKPROTECTOR && CURRENT_POINTER_IN_TPIDRURO && !XIP_DEFLATED_DATA + depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS + select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS default y help Due to the fact that GCC uses an ordinary symbol reference from diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index 976315dea958..0c9497d549e3 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -65,7 +65,7 @@ config UNWINDER_FRAME_POINTER config UNWINDER_ARM bool "ARM EABI stack unwinder" - depends on AEABI && !FUNCTION_GRAPH_TRACER + depends on AEABI select ARM_UNWIND help This option enables stack unwinding support in the kernel diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 77172d555c7e..e943624cbf87 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -275,6 +275,14 @@ endif ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) prepare: stack_protector_prepare +ifeq ($(CONFIG_CC_HAVE_STACKPROTECTOR_TLS),y) +stack_protector_prepare: prepare0 + $(eval KBUILD_CFLAGS += \ + -mstack-protector-guard=tls \ + -mstack-protector-guard-offset=$(shell \ + awk '{if ($$2 == "TSK_STACK_CANARY") print $$3;}'\ + include/generated/asm-offsets.h)) +else stack_protector_prepare: prepare0 $(eval SSP_PLUGIN_CFLAGS := \ -fplugin-arg-arm_ssp_per_task_plugin-offset=$(shell \ @@ -283,6 +291,7 @@ stack_protector_prepare: prepare0 $(eval KBUILD_CFLAGS += $(SSP_PLUGIN_CFLAGS)) $(eval GCC_PLUGINS_CFLAGS += $(SSP_PLUGIN_CFLAGS)) endif +endif all: $(notdir $(KBUILD_IMAGE)) diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 954eee8a785a..187a187706cb 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -92,17 +92,13 @@ ifeq ($(CONFIG_USE_OF),y) OBJS += $(libfdt_objs) fdt_check_mem_start.o endif -# -fstack-protector-strong triggers protection checks in this code, -# but it is being used too early to link to meaningful stack_chk logic. -$(foreach o, $(libfdt_objs) atags_to_fdt.o fdt_check_mem_start.o, \ - $(eval CFLAGS_$(o) := -I $(srctree)/scripts/dtc/libfdt -fno-stack-protector)) - targets := vmlinux vmlinux.lds piggy_data piggy.o \ head.o $(OBJS) KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING ccflags-y := -fpic $(call cc-option,-mno-single-pic-base,) -fno-builtin \ + -I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \ -I$(obj) $(DISABLE_ARM_SSP_PER_TASK_PLUGIN) ccflags-remove-$(CONFIG_FUNCTION_TRACER) += -pg asflags-y := -DZIMAGE diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c index e1e9a5dde853..c3c66ff2d696 100644 --- a/arch/arm/boot/compressed/misc.c +++ b/arch/arm/boot/compressed/misc.c @@ -128,13 +128,6 @@ asmlinkage void __div0(void) error("Attempting division by 0!"); } -const unsigned long __stack_chk_guard = 0x000a0dff; - -void __stack_chk_fail(void) -{ - error("stack-protector: Kernel stack is corrupted\n"); -} - extern int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x)); diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 235ad559acb2..e41eca79c950 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -806,6 +806,7 @@ dtb-$(CONFIG_ARCH_OMAP3) += \ logicpd-som-lv-37xx-devkit.dtb \ omap3430-sdp.dtb \ omap3-beagle.dtb \ + omap3-beagle-ab4.dtb \ omap3-beagle-xm.dtb \ omap3-beagle-xm-ab.dtb \ omap3-cm-t3517.dtb \ diff --git a/arch/arm/boot/dts/am335x-wega.dtsi b/arch/arm/boot/dts/am335x-wega.dtsi index 673159d93a6a..f957fea8208e 100644 --- a/arch/arm/boot/dts/am335x-wega.dtsi +++ b/arch/arm/boot/dts/am335x-wega.dtsi @@ -55,7 +55,7 @@ 2 1 0 0 /* # 0: INACTIVE, 1: TX, 2: RX */ >; tx-num-evt = <16>; - rt-num-evt = <16>; + rx-num-evt = <16>; status = "okay"; }; diff --git a/arch/arm/boot/dts/am43x-epos-evm.dts b/arch/arm/boot/dts/am43x-epos-evm.dts index 2f4d2e4e9b3e..4f9a7251a107 100644 --- a/arch/arm/boot/dts/am43x-epos-evm.dts +++ b/arch/arm/boot/dts/am43x-epos-evm.dts @@ -11,7 +11,7 @@ #include <dt-bindings/pinctrl/am43xx.h> #include <dt-bindings/gpio/gpio.h> #include <dt-bindings/pwm/pwm.h> -#include <dt-bindings/sound/tlv320aic31xx-micbias.h> +#include <dt-bindings/sound/tlv320aic31xx.h> / { model = "TI AM43x EPOS EVM"; diff --git a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi index 6dde51c2aed3..e4775bbceecc 100644 --- a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi +++ b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi @@ -118,7 +118,7 @@ }; pinctrl_fwqspid_default: fwqspid_default { - function = "FWQSPID"; + function = "FWSPID"; groups = "FWQSPID"; }; diff --git a/arch/arm/boot/dts/bcm2711.dtsi b/arch/arm/boot/dts/bcm2711.dtsi index dff18fc9a906..21294f775a20 100644 --- a/arch/arm/boot/dts/bcm2711.dtsi +++ b/arch/arm/boot/dts/bcm2711.dtsi @@ -290,6 +290,7 @@ hvs: hvs@7e400000 { compatible = "brcm,bcm2711-hvs"; + reg = <0x7e400000 0x8000>; interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>; }; diff --git a/arch/arm/boot/dts/dra7-l4.dtsi b/arch/arm/boot/dts/dra7-l4.dtsi index 956a26d52a4c..0a11bacffc1f 100644 --- a/arch/arm/boot/dts/dra7-l4.dtsi +++ b/arch/arm/boot/dts/dra7-l4.dtsi @@ -3482,8 +3482,7 @@ ti,timer-pwm; }; }; - - target-module@2c000 { /* 0x4882c000, ap 17 02.0 */ + timer15_target: target-module@2c000 { /* 0x4882c000, ap 17 02.0 */ compatible = "ti,sysc-omap4-timer", "ti,sysc"; reg = <0x2c000 0x4>, <0x2c010 0x4>; @@ -3511,7 +3510,7 @@ }; }; - target-module@2e000 { /* 0x4882e000, ap 19 14.0 */ + timer16_target: target-module@2e000 { /* 0x4882e000, ap 19 14.0 */ compatible = "ti,sysc-omap4-timer", "ti,sysc"; reg = <0x2e000 0x4>, <0x2e010 0x4>; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 6b485cbed8d5..97ce0c4f1df7 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -160,7 +160,7 @@ target-module@48210000 { compatible = "ti,sysc-omap4-simple", "ti,sysc"; power-domains = <&prm_mpu>; - clocks = <&mpu_clkctrl DRA7_MPU_CLKCTRL 0>; + clocks = <&mpu_clkctrl DRA7_MPU_MPU_CLKCTRL 0>; clock-names = "fck"; #address-cells = <1>; #size-cells = <1>; @@ -875,10 +875,10 @@ <0x58000014 4>; reg-names = "rev", "syss"; ti,syss-mask = <1>; - clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 0>, - <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 9>, - <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 10>, - <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 11>; + clocks = <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 0>, + <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 9>, + <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 10>, + <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 11>; clock-names = "fck", "hdmi_clk", "sys_clk", "tv_clk"; #address-cells = <1>; #size-cells = <1>; @@ -912,7 +912,7 @@ SYSC_OMAP2_SOFTRESET | SYSC_OMAP2_AUTOIDLE)>; ti,syss-mask = <1>; - clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 8>; + clocks = <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 8>; clock-names = "fck"; #address-cells = <1>; #size-cells = <1>; @@ -939,8 +939,8 @@ <SYSC_IDLE_SMART>, <SYSC_IDLE_SMART_WKUP>; ti,sysc-mask = <(SYSC_OMAP4_SOFTRESET)>; - clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 9>, - <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 8>; + clocks = <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 9>, + <&dss_clkctrl DRA7_DSS_DSS_CORE_CLKCTRL 8>; clock-names = "fck", "dss_clk"; #address-cells = <1>; #size-cells = <1>; @@ -979,7 +979,7 @@ compatible = "vivante,gc"; reg = <0x0 0x700>; interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&dss_clkctrl DRA7_BB2D_CLKCTRL 0>; + clocks = <&dss_clkctrl DRA7_DSS_BB2D_CLKCTRL 0>; clock-names = "core"; }; }; @@ -1333,26 +1333,26 @@ ti,no-reset-on-init; ti,no-idle; timer@0 { - assigned-clocks = <&wkupaon_clkctrl DRA7_TIMER1_CLKCTRL 24>; + assigned-clocks = <&wkupaon_clkctrl DRA7_WKUPAON_TIMER1_CLKCTRL 24>; assigned-clock-parents = <&sys_32k_ck>; }; }; /* Local timers, see ARM architected timer wrap erratum i940 */ -&timer3_target { +&timer15_target { ti,no-reset-on-init; ti,no-idle; timer@0 { - assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER3_CLKCTRL 24>; + assigned-clocks = <&l4per3_clkctrl DRA7_L4PER3_TIMER15_CLKCTRL 24>; assigned-clock-parents = <&timer_sys_clk_div>; }; }; -&timer4_target { +&timer16_target { ti,no-reset-on-init; ti,no-idle; timer@0 { - assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER4_CLKCTRL 24>; + assigned-clocks = <&l4per3_clkctrl DRA7_L4PER3_TIMER16_CLKCTRL 24>; assigned-clock-parents = <&timer_sys_clk_div>; }; }; diff --git a/arch/arm/boot/dts/imx23-evk.dts b/arch/arm/boot/dts/imx23-evk.dts index 8cbaf1c81174..3b609d987d88 100644 --- a/arch/arm/boot/dts/imx23-evk.dts +++ b/arch/arm/boot/dts/imx23-evk.dts @@ -79,7 +79,6 @@ MX23_PAD_LCD_RESET__GPIO_1_18 MX23_PAD_PWM3__GPIO_1_29 MX23_PAD_PWM4__GPIO_1_30 - MX23_PAD_SSP1_DETECT__SSP1_DETECT >; fsl,drive-strength = <MXS_DRIVE_4mA>; fsl,voltage = <MXS_VOLTAGE_HIGH>; diff --git a/arch/arm/boot/dts/imx6qdl-udoo.dtsi b/arch/arm/boot/dts/imx6qdl-udoo.dtsi index d07d8f83456d..ccfa8e320be6 100644 --- a/arch/arm/boot/dts/imx6qdl-udoo.dtsi +++ b/arch/arm/boot/dts/imx6qdl-udoo.dtsi @@ -5,6 +5,8 @@ * Author: Fabio Estevam <fabio.estevam@freescale.com> */ +#include <dt-bindings/gpio/gpio.h> + / { aliases { backlight = &backlight; @@ -226,6 +228,7 @@ MX6QDL_PAD_SD3_DAT1__SD3_DATA1 0x17059 MX6QDL_PAD_SD3_DAT2__SD3_DATA2 0x17059 MX6QDL_PAD_SD3_DAT3__SD3_DATA3 0x17059 + MX6QDL_PAD_SD3_DAT5__GPIO7_IO00 0x1b0b0 >; }; @@ -304,7 +307,7 @@ &usdhc3 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usdhc3>; - non-removable; + cd-gpios = <&gpio7 0 GPIO_ACTIVE_LOW>; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi index b7ea37ad4e55..bcec98b96411 100644 --- a/arch/arm/boot/dts/imx7ulp.dtsi +++ b/arch/arm/boot/dts/imx7ulp.dtsi @@ -259,7 +259,7 @@ interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>; clocks = <&pcc2 IMX7ULP_CLK_WDG1>; assigned-clocks = <&pcc2 IMX7ULP_CLK_WDG1>; - assigned-clocks-parents = <&scg1 IMX7ULP_CLK_FIRC_BUS_CLK>; + assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC_BUS_CLK>; timeout-sec = <40>; }; diff --git a/arch/arm/boot/dts/meson.dtsi b/arch/arm/boot/dts/meson.dtsi index 3be7cba603d5..26eaba3fa96f 100644 --- a/arch/arm/boot/dts/meson.dtsi +++ b/arch/arm/boot/dts/meson.dtsi @@ -59,7 +59,7 @@ }; uart_A: serial@84c0 { - compatible = "amlogic,meson6-uart", "amlogic,meson-uart"; + compatible = "amlogic,meson6-uart"; reg = <0x84c0 0x18>; interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>; fifo-size = <128>; @@ -67,7 +67,7 @@ }; uart_B: serial@84dc { - compatible = "amlogic,meson6-uart", "amlogic,meson-uart"; + compatible = "amlogic,meson6-uart"; reg = <0x84dc 0x18>; interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>; status = "disabled"; @@ -105,7 +105,7 @@ }; uart_C: serial@8700 { - compatible = "amlogic,meson6-uart", "amlogic,meson-uart"; + compatible = "amlogic,meson6-uart"; reg = <0x8700 0x18>; interrupts = <GIC_SPI 93 IRQ_TYPE_EDGE_RISING>; status = "disabled"; @@ -228,7 +228,7 @@ }; uart_AO: serial@4c0 { - compatible = "amlogic,meson6-uart", "amlogic,meson-ao-uart", "amlogic,meson-uart"; + compatible = "amlogic,meson6-uart", "amlogic,meson-ao-uart"; reg = <0x4c0 0x18>; interrupts = <GIC_SPI 90 IRQ_TYPE_EDGE_RISING>; status = "disabled"; diff --git a/arch/arm/boot/dts/meson8.dtsi b/arch/arm/boot/dts/meson8.dtsi index f80ddc98d3a2..9997a5d0333a 100644 --- a/arch/arm/boot/dts/meson8.dtsi +++ b/arch/arm/boot/dts/meson8.dtsi @@ -736,27 +736,27 @@ }; &uart_AO { - compatible = "amlogic,meson8-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_CLK81>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8-uart", "amlogic,meson-ao-uart"; + clocks = <&xtal>, <&clkc CLKID_CLK81>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_A { - compatible = "amlogic,meson8-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART0>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_B { - compatible = "amlogic,meson8-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART1>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_C { - compatible = "amlogic,meson8-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART2>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &usb0 { diff --git a/arch/arm/boot/dts/meson8b.dtsi b/arch/arm/boot/dts/meson8b.dtsi index b49b7cbaed4e..94f1c03decce 100644 --- a/arch/arm/boot/dts/meson8b.dtsi +++ b/arch/arm/boot/dts/meson8b.dtsi @@ -724,27 +724,27 @@ }; &uart_AO { - compatible = "amlogic,meson8b-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_CLK81>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8b-uart", "amlogic,meson-ao-uart"; + clocks = <&xtal>, <&clkc CLKID_CLK81>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_A { - compatible = "amlogic,meson8b-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART0>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8b-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_B { - compatible = "amlogic,meson8b-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART1>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8b-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &uart_C { - compatible = "amlogic,meson8b-uart", "amlogic,meson-uart"; - clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART2>; - clock-names = "baud", "xtal", "pclk"; + compatible = "amlogic,meson8b-uart"; + clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>; + clock-names = "xtal", "pclk", "baud"; }; &usb0 { diff --git a/arch/arm/boot/dts/omap3-beagle-ab4.dts b/arch/arm/boot/dts/omap3-beagle-ab4.dts new file mode 100644 index 000000000000..990ff2d84686 --- /dev/null +++ b/arch/arm/boot/dts/omap3-beagle-ab4.dts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only +/dts-v1/; + +#include "omap3-beagle.dts" + +/ { + model = "TI OMAP3 BeagleBoard A to B4"; + compatible = "ti,omap3-beagle-ab4", "ti,omap3-beagle", "ti,omap3430", "ti,omap3"; +}; + +/* + * Workaround for capacitor C70 issue, see "Boards revision A and < B5" + * section at https://elinux.org/BeagleBoard_Community + */ + +/* Unusable as clocksource because of unreliable oscillator */ +&counter32k { + status = "disabled"; +}; + +/* Unusable as clockevent because of unreliable oscillator, allow to idle */ +&timer1_target { + /delete-property/ti,no-reset-on-init; + /delete-property/ti,no-idle; + timer@0 { + /delete-property/ti,timer-alwon; + }; +}; + +/* Preferred always-on timer for clocksource */ +&timer12_target { + ti,no-reset-on-init; + ti,no-idle; + timer@0 { + /* Always clocked by secure_32k_fck */ + }; +}; + +/* Preferred timer for clockevent */ +&timer2_target { + ti,no-reset-on-init; + ti,no-idle; + timer@0 { + assigned-clocks = <&gpt2_fck>; + assigned-clock-parents = <&sys_ck>; + }; +}; diff --git a/arch/arm/boot/dts/omap3-beagle.dts b/arch/arm/boot/dts/omap3-beagle.dts index f9f34b8458e9..0548b391334f 100644 --- a/arch/arm/boot/dts/omap3-beagle.dts +++ b/arch/arm/boot/dts/omap3-beagle.dts @@ -304,39 +304,6 @@ phys = <0 &hsusb2_phy>; }; -/* Unusable as clocksource because of unreliable oscillator */ -&counter32k { - status = "disabled"; -}; - -/* Unusable as clockevent because if unreliable oscillator, allow to idle */ -&timer1_target { - /delete-property/ti,no-reset-on-init; - /delete-property/ti,no-idle; - timer@0 { - /delete-property/ti,timer-alwon; - }; -}; - -/* Preferred always-on timer for clocksource */ -&timer12_target { - ti,no-reset-on-init; - ti,no-idle; - timer@0 { - /* Always clocked by secure_32k_fck */ - }; -}; - -/* Preferred timer for clockevent */ -&timer2_target { - ti,no-reset-on-init; - ti,no-idle; - timer@0 { - assigned-clocks = <&gpt2_fck>; - assigned-clock-parents = <&sys_ck>; - }; -}; - &twl_gpio { ti,use-leds; /* pullups: BIT(1) */ diff --git a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi index 5e55198e4576..54cd37336be7 100644 --- a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi +++ b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi @@ -158,6 +158,24 @@ status = "disabled"; }; +/* Unusable as clockevent because if unreliable oscillator, allow to idle */ +&timer1_target { + /delete-property/ti,no-reset-on-init; + /delete-property/ti,no-idle; + timer@0 { + /delete-property/ti,timer-alwon; + }; +}; + +/* Preferred timer for clockevent */ +&timer12_target { + ti,no-reset-on-init; + ti,no-idle; + timer@0 { + /* Always clocked by secure_32k_fck */ + }; +}; + &twl_gpio { ti,use-leds; /* diff --git a/arch/arm/boot/dts/omap3-devkit8000.dts b/arch/arm/boot/dts/omap3-devkit8000.dts index c2995a280729..162d0726b008 100644 --- a/arch/arm/boot/dts/omap3-devkit8000.dts +++ b/arch/arm/boot/dts/omap3-devkit8000.dts @@ -14,36 +14,3 @@ display2 = &tv0; }; }; - -/* Unusable as clocksource because of unreliable oscillator */ -&counter32k { - status = "disabled"; -}; - -/* Unusable as clockevent because if unreliable oscillator, allow to idle */ -&timer1_target { - /delete-property/ti,no-reset-on-init; - /delete-property/ti,no-idle; - timer@0 { - /delete-property/ti,timer-alwon; - }; -}; - -/* Preferred always-on timer for clocksource */ -&timer12_target { - ti,no-reset-on-init; - ti,no-idle; - timer@0 { - /* Always clocked by secure_32k_fck */ - }; -}; - -/* Preferred timer for clockevent */ -&timer2_target { - ti,no-reset-on-init; - ti,no-idle; - timer@0 { - assigned-clocks = <&gpt2_fck>; - assigned-clock-parents = <&sys_ck>; - }; -}; diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi index 8eed9e3a92e9..5868eb512f69 100644 --- a/arch/arm/boot/dts/rk322x.dtsi +++ b/arch/arm/boot/dts/rk322x.dtsi @@ -718,8 +718,8 @@ interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>; assigned-clocks = <&cru SCLK_HDMI_PHY>; assigned-clock-parents = <&hdmi_phy>; - clocks = <&cru SCLK_HDMI_HDCP>, <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_CEC>; - clock-names = "isfr", "iahb", "cec"; + clocks = <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_HDCP>, <&cru SCLK_HDMI_CEC>; + clock-names = "iahb", "isfr", "cec"; pinctrl-names = "default"; pinctrl-0 = <&hdmii2c_xfer &hdmi_hpd &hdmi_cec>; resets = <&cru SRST_HDMI_P>; diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi index aaaa61875701..45a9d9b908d2 100644 --- a/arch/arm/boot/dts/rk3288.dtsi +++ b/arch/arm/boot/dts/rk3288.dtsi @@ -971,7 +971,7 @@ status = "disabled"; }; - crypto: cypto-controller@ff8a0000 { + crypto: crypto@ff8a0000 { compatible = "rockchip,rk3288-crypto"; reg = <0x0 0xff8a0000 0x0 0x4000>; interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>; diff --git a/arch/arm/boot/dts/spear320-hmi.dts b/arch/arm/boot/dts/spear320-hmi.dts index 367ba48aac3e..b587e4ec11e5 100644 --- a/arch/arm/boot/dts/spear320-hmi.dts +++ b/arch/arm/boot/dts/spear320-hmi.dts @@ -235,7 +235,6 @@ #address-cells = <1>; #size-cells = <0>; reg = <0x41>; - irq-over-gpio; irq-gpios = <&gpiopinctrl 29 0x4>; id = <0>; blocks = <0x5>; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts index 580ca497f312..f8c5899fbdba 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts @@ -185,10 +185,6 @@ cap-sd-highspeed; cap-mmc-highspeed; /* All direction control is used */ - st,sig-dir-cmd; - st,sig-dir-dat0; - st,sig-dir-dat2; - st,sig-dir-dat31; st,sig-pin-fbclk; full-pwr-cycle; vmmc-supply = <&ab8500_ldo_aux3_reg>; diff --git a/arch/arm/boot/dts/tegra124-nyan-big-fhd.dts b/arch/arm/boot/dts/tegra124-nyan-big-fhd.dts index d35fb79d2f51..4db43324dafa 100644 --- a/arch/arm/boot/dts/tegra124-nyan-big-fhd.dts +++ b/arch/arm/boot/dts/tegra124-nyan-big-fhd.dts @@ -5,7 +5,13 @@ / { /* Version of Nyan Big with 1080p panel */ - panel { - compatible = "auo,b133htn01"; + host1x@50000000 { + dpaux@545c0000 { + aux-bus { + panel: panel { + compatible = "auo,b133htn01"; + }; + }; + }; }; }; diff --git a/arch/arm/boot/dts/tegra124-nyan-big.dts b/arch/arm/boot/dts/tegra124-nyan-big.dts index 1d2aac2cb6d0..fdc1d64dfff9 100644 --- a/arch/arm/boot/dts/tegra124-nyan-big.dts +++ b/arch/arm/boot/dts/tegra124-nyan-big.dts @@ -13,12 +13,15 @@ "google,nyan-big-rev1", "google,nyan-big-rev0", "google,nyan-big", "google,nyan", "nvidia,tegra124"; - panel: panel { - compatible = "auo,b133xtn01"; - - power-supply = <&vdd_3v3_panel>; - backlight = <&backlight>; - ddc-i2c-bus = <&dpaux>; + host1x@50000000 { + dpaux@545c0000 { + aux-bus { + panel: panel { + compatible = "auo,b133xtn01"; + backlight = <&backlight>; + }; + }; + }; }; mmc@700b0400 { /* SD Card on this bus */ diff --git a/arch/arm/boot/dts/tegra124-nyan-blaze.dts b/arch/arm/boot/dts/tegra124-nyan-blaze.dts index 677babde6460..abdf4456826f 100644 --- a/arch/arm/boot/dts/tegra124-nyan-blaze.dts +++ b/arch/arm/boot/dts/tegra124-nyan-blaze.dts @@ -15,12 +15,15 @@ "google,nyan-blaze-rev0", "google,nyan-blaze", "google,nyan", "nvidia,tegra124"; - panel: panel { - compatible = "samsung,ltn140at29-301"; - - power-supply = <&vdd_3v3_panel>; - backlight = <&backlight>; - ddc-i2c-bus = <&dpaux>; + host1x@50000000 { + dpaux@545c0000 { + aux-bus { + panel: panel { + compatible = "samsung,ltn140at29-301"; + backlight = <&backlight>; + }; + }; + }; }; sound { diff --git a/arch/arm/boot/dts/tegra124-venice2.dts b/arch/arm/boot/dts/tegra124-venice2.dts index 232c90604df9..6a9592ceb5f2 100644 --- a/arch/arm/boot/dts/tegra124-venice2.dts +++ b/arch/arm/boot/dts/tegra124-venice2.dts @@ -48,6 +48,13 @@ dpaux@545c0000 { vdd-supply = <&vdd_3v3_panel>; status = "okay"; + + aux-bus { + panel: panel { + compatible = "lg,lp129qe"; + backlight = <&backlight>; + }; + }; }; }; @@ -1080,13 +1087,6 @@ }; }; - panel: panel { - compatible = "lg,lp129qe"; - power-supply = <&vdd_3v3_panel>; - backlight = <&backlight>; - ddc-i2c-bus = <&dpaux>; - }; - vdd_mux: regulator-mux { compatible = "regulator-fixed"; regulator-name = "+VDD_MUX"; diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig index e849367c0566..b58d45a03607 100644 --- a/arch/arm/configs/davinci_all_defconfig +++ b/arch/arm/configs/davinci_all_defconfig @@ -258,7 +258,6 @@ CONFIG_MINIX_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=m CONFIG_NLS_ISO8859_1=y diff --git a/arch/arm/configs/ezx_defconfig b/arch/arm/configs/ezx_defconfig index ec84d80096b1..0788a892e160 100644 --- a/arch/arm/configs/ezx_defconfig +++ b/arch/arm/configs/ezx_defconfig @@ -309,7 +309,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_NFS_V3_ACL=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y CONFIG_SMB_FS=m CONFIG_CIFS=m diff --git a/arch/arm/configs/imote2_defconfig b/arch/arm/configs/imote2_defconfig index 6db871d4e077..015b7ef237de 100644 --- a/arch/arm/configs/imote2_defconfig +++ b/arch/arm/configs/imote2_defconfig @@ -283,7 +283,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_NFS_V3_ACL=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y CONFIG_SMB_FS=m CONFIG_CIFS=m diff --git a/arch/arm/configs/integrator_defconfig b/arch/arm/configs/integrator_defconfig index 4dfe321a79f6..5b485722ccf9 100644 --- a/arch/arm/configs/integrator_defconfig +++ b/arch/arm/configs/integrator_defconfig @@ -79,7 +79,6 @@ CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/arm/configs/iop32x_defconfig b/arch/arm/configs/iop32x_defconfig index 18a21faa834c..de2cc6759cae 100644 --- a/arch/arm/configs/iop32x_defconfig +++ b/arch/arm/configs/iop32x_defconfig @@ -94,7 +94,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_PARTITION_ADVANCED=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/arm/configs/keystone_defconfig b/arch/arm/configs/keystone_defconfig index 33c917df7b32..b1bcb858216b 100644 --- a/arch/arm/configs/keystone_defconfig +++ b/arch/arm/configs/keystone_defconfig @@ -213,7 +213,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/arm/configs/lart_defconfig b/arch/arm/configs/lart_defconfig index b6ddb9884326..2dfa33d7dca3 100644 --- a/arch/arm/configs/lart_defconfig +++ b/arch/arm/configs/lart_defconfig @@ -59,7 +59,6 @@ CONFIG_CRAMFS=m CONFIG_NFS_FS=m CONFIG_NFS_V3=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NLS=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_850=m diff --git a/arch/arm/configs/netwinder_defconfig b/arch/arm/configs/netwinder_defconfig index 2e3b20ef0db1..c3c171cec91b 100644 --- a/arch/arm/configs/netwinder_defconfig +++ b/arch/arm/configs/netwinder_defconfig @@ -71,7 +71,6 @@ CONFIG_NFS_V3=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_SMB_FS=y CONFIG_PARTITION_ADVANCED=y CONFIG_NLS_CODEPAGE_437=y diff --git a/arch/arm/configs/versatile_defconfig b/arch/arm/configs/versatile_defconfig index d06aa64e05a1..c2d79a67f81f 100644 --- a/arch/arm/configs/versatile_defconfig +++ b/arch/arm/configs/versatile_defconfig @@ -86,7 +86,6 @@ CONFIG_ROMFS_FS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_850=m CONFIG_NLS_ISO8859_1=m CONFIG_MAGIC_SYSRQ=y diff --git a/arch/arm/configs/viper_defconfig b/arch/arm/configs/viper_defconfig index 989599ce5300..c28539bfd128 100644 --- a/arch/arm/configs/viper_defconfig +++ b/arch/arm/configs/viper_defconfig @@ -145,7 +145,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_PARTITION_ADVANCED=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_850=m diff --git a/arch/arm/configs/zeus_defconfig b/arch/arm/configs/zeus_defconfig index d3b98c4d225b..25bb6995f105 100644 --- a/arch/arm/configs/zeus_defconfig +++ b/arch/arm/configs/zeus_defconfig @@ -160,7 +160,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_PARTITION_ADVANCED=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_850=m diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S index 7d0cc7f226a5..7b61032f29fa 100644 --- a/arch/arm/crypto/aes-neonbs-core.S +++ b/arch/arm/crypto/aes-neonbs-core.S @@ -758,29 +758,24 @@ ENTRY(aesbs_cbc_decrypt) ENDPROC(aesbs_cbc_decrypt) .macro next_ctr, q - vmov.32 \q\()h[1], r10 + vmov \q\()h, r9, r10 adds r10, r10, #1 - vmov.32 \q\()h[0], r9 adcs r9, r9, #0 - vmov.32 \q\()l[1], r8 + vmov \q\()l, r7, r8 adcs r8, r8, #0 - vmov.32 \q\()l[0], r7 adc r7, r7, #0 vrev32.8 \q, \q .endm /* * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], - * int rounds, int blocks, u8 ctr[], u8 final[]) + * int rounds, int bytes, u8 ctr[]) */ ENTRY(aesbs_ctr_encrypt) mov ip, sp push {r4-r10, lr} - ldm ip, {r5-r7} // load args 4-6 - teq r7, #0 - addne r5, r5, #1 // one extra block if final != 0 - + ldm ip, {r5, r6} // load args 4-5 vld1.8 {q0}, [r6] // load counter vrev32.8 q1, q0 vmov r9, r10, d3 @@ -792,20 +787,19 @@ ENTRY(aesbs_ctr_encrypt) adc r7, r7, #0 99: vmov q1, q0 + sub lr, r5, #1 vmov q2, q0 + adr ip, 0f vmov q3, q0 + and lr, lr, #112 vmov q4, q0 + cmp r5, #112 vmov q5, q0 + sub ip, ip, lr, lsl #1 vmov q6, q0 + add ip, ip, lr, lsr #2 vmov q7, q0 - - adr ip, 0f - sub lr, r5, #1 - and lr, lr, #7 - cmp r5, #8 - sub ip, ip, lr, lsl #5 - sub ip, ip, lr, lsl #2 - movlt pc, ip // computed goto if blocks < 8 + movle pc, ip // computed goto if bytes < 112 next_ctr q1 next_ctr q2 @@ -820,12 +814,14 @@ ENTRY(aesbs_ctr_encrypt) bl aesbs_encrypt8 adr ip, 1f - and lr, r5, #7 - cmp r5, #8 - movgt r4, #0 - ldrle r4, [sp, #40] // load final in the last round - sub ip, ip, lr, lsl #2 - movlt pc, ip // computed goto if blocks < 8 + sub lr, r5, #1 + cmp r5, #128 + bic lr, lr, #15 + ands r4, r5, #15 // preserves C flag + teqcs r5, r5 // set Z flag if not last iteration + sub ip, ip, lr, lsr #2 + rsb r4, r4, #16 + movcc pc, ip // computed goto if bytes < 128 vld1.8 {q8}, [r1]! vld1.8 {q9}, [r1]! @@ -834,46 +830,70 @@ ENTRY(aesbs_ctr_encrypt) vld1.8 {q12}, [r1]! vld1.8 {q13}, [r1]! vld1.8 {q14}, [r1]! - teq r4, #0 // skip last block if 'final' -1: bne 2f +1: subne r1, r1, r4 vld1.8 {q15}, [r1]! -2: adr ip, 3f - cmp r5, #8 - sub ip, ip, lr, lsl #3 - movlt pc, ip // computed goto if blocks < 8 + add ip, ip, #2f - 1b veor q0, q0, q8 - vst1.8 {q0}, [r0]! veor q1, q1, q9 - vst1.8 {q1}, [r0]! veor q4, q4, q10 - vst1.8 {q4}, [r0]! veor q6, q6, q11 - vst1.8 {q6}, [r0]! veor q3, q3, q12 - vst1.8 {q3}, [r0]! veor q7, q7, q13 - vst1.8 {q7}, [r0]! veor q2, q2, q14 + bne 3f + veor q5, q5, q15 + + movcc pc, ip // computed goto if bytes < 128 + + vst1.8 {q0}, [r0]! + vst1.8 {q1}, [r0]! + vst1.8 {q4}, [r0]! + vst1.8 {q6}, [r0]! + vst1.8 {q3}, [r0]! + vst1.8 {q7}, [r0]! vst1.8 {q2}, [r0]! - teq r4, #0 // skip last block if 'final' - W(bne) 5f -3: veor q5, q5, q15 +2: subne r0, r0, r4 vst1.8 {q5}, [r0]! -4: next_ctr q0 + next_ctr q0 - subs r5, r5, #8 + subs r5, r5, #128 bgt 99b vst1.8 {q0}, [r6] pop {r4-r10, pc} -5: vst1.8 {q5}, [r4] - b 4b +3: adr lr, .Lpermute_table + 16 + cmp r5, #16 // Z flag remains cleared + sub lr, lr, r4 + vld1.8 {q8-q9}, [lr] + vtbl.8 d16, {q5}, d16 + vtbl.8 d17, {q5}, d17 + veor q5, q8, q15 + bcc 4f // have to reload prev if R5 < 16 + vtbx.8 d10, {q2}, d18 + vtbx.8 d11, {q2}, d19 + mov pc, ip // branch back to VST sequence + +4: sub r0, r0, r4 + vshr.s8 q9, q9, #7 // create mask for VBIF + vld1.8 {q8}, [r0] // reload + vbif q5, q8, q9 + vst1.8 {q5}, [r0] + pop {r4-r10, pc} ENDPROC(aesbs_ctr_encrypt) + .align 6 +.Lpermute_table: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .macro next_tweak, out, in, const, tmp vshr.s64 \tmp, \in, #63 vand \tmp, \tmp, \const @@ -888,6 +908,7 @@ ENDPROC(aesbs_ctr_encrypt) * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, * int blocks, u8 iv[], int reorder_last_tweak) */ + .align 6 __xts_prepare8: vld1.8 {q14}, [r7] // load iv vmov.i32 d30, #0x87 // compose tweak mask vector diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 5c6cd3c63cbc..f00f042ef357 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -37,7 +37,7 @@ asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], - int rounds, int blocks, u8 ctr[], u8 final[]); + int rounds, int blocks, u8 ctr[]); asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[], int); @@ -243,32 +243,25 @@ static int ctr_encrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes > 0) { - unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; - u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + int bytes = walk.nbytes; - if (walk.nbytes < walk.total) { - blocks = round_down(blocks, - walk.stride / AES_BLOCK_SIZE); - final = NULL; - } + if (unlikely(bytes < AES_BLOCK_SIZE)) + src = dst = memcpy(buf + sizeof(buf) - bytes, + src, bytes); + else if (walk.nbytes < walk.total) + bytes &= ~(8 * AES_BLOCK_SIZE - 1); kernel_neon_begin(); - aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, - ctx->rk, ctx->rounds, blocks, walk.iv, final); + aesbs_ctr_encrypt(dst, src, ctx->rk, ctx->rounds, bytes, walk.iv); kernel_neon_end(); - if (final) { - u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; - u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; + if (unlikely(bytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - bytes, bytes); - crypto_xor_cpy(dst, src, final, - walk.total % AES_BLOCK_SIZE); - - err = skcipher_walk_done(&walk, 0); - break; - } - err = skcipher_walk_done(&walk, - walk.nbytes - blocks * AES_BLOCK_SIZE); + err = skcipher_walk_done(&walk, walk.nbytes - bytes); } return err; diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 6fe67963ba5a..34fe8d2dd5d1 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -86,6 +86,10 @@ #define IMM12_MASK 0xfff +/* the frame pointer used for stack unwinding */ +ARM( fpreg .req r11 ) +THUMB( fpreg .req r7 ) + /* * Enable and disable interrupts */ @@ -107,6 +111,16 @@ .endm #endif +#if __LINUX_ARM_ARCH__ < 7 + .macro dsb, args + mcr p15, 0, r0, c7, c10, 4 + .endm + + .macro isb, args + mcr p15, 0, r0, c7, c5, 4 + .endm +#endif + .macro asm_trace_hardirqs_off, save=1 #if defined(CONFIG_TRACE_IRQFLAGS) .if \save @@ -199,43 +213,12 @@ .endm .endr - .macro get_current, rd -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register -#else - get_thread_info \rd - ldr \rd, [\rd, #TI_TASK] -#endif - .endm - - .macro set_current, rn -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register -#endif - .endm - - .macro reload_current, t1:req, t2:req -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - adr_l \t1, __entry_task @ get __entry_task base address - mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset - ldr \t1, [\t1, \t2] @ load variable - mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO -#endif - .endm - /* * Get current thread_info. */ .macro get_thread_info, rd -#ifdef CONFIG_THREAD_INFO_IN_TASK /* thread_info is the first member of struct task_struct */ get_current \rd -#else - ARM( mov \rd, sp, lsr #THREAD_SIZE_ORDER + PAGE_SHIFT ) - THUMB( mov \rd, sp ) - THUMB( lsr \rd, \rd, #THREAD_SIZE_ORDER + PAGE_SHIFT ) - mov \rd, \rd, lsl #THREAD_SIZE_ORDER + PAGE_SHIFT -#endif .endm /* @@ -310,6 +293,80 @@ #define ALT_UP_B(label) b label #endif + /* + * this_cpu_offset - load the per-CPU offset of this CPU into + * register 'rd' + */ + .macro this_cpu_offset, rd:req +#ifdef CONFIG_SMP +ALT_SMP(mrc p15, 0, \rd, c13, c0, 4) +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L1_\@) +.L0_\@: + .subsection 1 +.L1_\@: ldr_va \rd, __per_cpu_offset + b .L0_\@ + .previous +#endif +#else + mov \rd, #0 +#endif + .endm + + /* + * set_current - store the task pointer of this CPU's current task + */ + .macro set_current, rn:req, tmp:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: str_va \rn, __current, \tmp + b .L1_\@ + .previous +.L1_\@: +#endif +#else + str_va \rn, __current, \tmp +#endif + .endm + + /* + * get_current - load the task pointer of this CPU's current task + */ + .macro get_current, rd:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: ldr_va \rd, __current + b .L1_\@ + .previous +.L1_\@: +#endif +#else + ldr_va \rd, __current +#endif + .endm + + /* + * reload_current - reload the task pointer of this CPU's current task + * into the TLS register + */ + .macro reload_current, t1:req, t2:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +#ifdef CONFIG_CPU_V6 +ALT_SMP(nop) +ALT_UP_B(.L0_\@) +#endif + ldr_this_cpu \t1, __entry_task, \t1, \t2 + mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO +.L0_\@: +#endif + .endm + /* * Instruction barrier */ @@ -566,12 +623,12 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) /* * mov_l - move a constant value or [relocated] address into a register */ - .macro mov_l, dst:req, imm:req + .macro mov_l, dst:req, imm:req, cond .if __LINUX_ARM_ARCH__ < 7 - ldr \dst, =\imm + ldr\cond \dst, =\imm .else - movw \dst, #:lower16:\imm - movt \dst, #:upper16:\imm + movw\cond \dst, #:lower16:\imm + movt\cond \dst, #:upper16:\imm .endif .endm @@ -609,6 +666,78 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __adldst_l str, \src, \sym, \tmp, \cond .endm + .macro __ldst_va, op, reg, tmp, sym, cond +#if __LINUX_ARM_ARCH__ >= 7 || \ + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + mov_l \tmp, \sym, \cond + \op\cond \reg, [\tmp] +#else + /* + * Avoid a literal load, by emitting a sequence of ADD/LDR instructions + * with the appropriate relocations. The combined sequence has a range + * of -/+ 256 MiB, which should be sufficient for the core kernel and + * for modules loaded into the module region. + */ + .globl \sym + .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym + .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym + .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym +.L0_\@: sub\cond \tmp, pc, #8 +.L1_\@: sub\cond \tmp, \tmp, #4 +.L2_\@: \op\cond \reg, [\tmp, #0] +#endif + .endm + + /* + * ldr_va - load a 32-bit word from the virtual address of \sym + */ + .macro ldr_va, rd:req, sym:req, cond + __ldst_va ldr, \rd, \rd, \sym, \cond + .endm + + /* + * str_va - store a 32-bit word to the virtual address of \sym + */ + .macro str_va, rn:req, sym:req, tmp:req, cond + __ldst_va str, \rn, \tmp, \sym, \cond + .endm + + /* + * ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym', + * without using a temp register. Supported in ARM mode + * only. + */ + .macro ldr_this_cpu_armv6, rd:req, sym:req + this_cpu_offset \rd + .globl \sym + .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym + .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym + .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym + add \rd, \rd, pc +.L0_\@: sub \rd, \rd, #4 +.L1_\@: sub \rd, \rd, #0 +.L2_\@: ldr \rd, [\rd, #4] + .endm + + /* + * ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym' + * into register 'rd', which may be the stack pointer, + * using 't1' and 't2' as general temp registers. These + * are permitted to overlap with 'rd' if != sp + */ + .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req +#if __LINUX_ARM_ARCH__ >= 7 || \ + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + this_cpu_offset \t1 + mov_l \t2, \sym + ldr \rd, [\t1, \t2] +#else + ldr_this_cpu_armv6 \rd, \sym +#endif + .endm + /* * rev_l - byte-swap a 32-bit value * @@ -626,4 +755,19 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .endif .endm + /* + * bl_r - branch and link to register + * + * @dst: target to branch to + * @c: conditional opcode suffix + */ + .macro bl_r, dst:req, c + .if __LINUX_ARM_ARCH__ < 6 + mov\c lr, pc + mov\c pc, \dst + .else + blx\c \dst + .endif + .endm + #endif /* __ASM_ASSEMBLER_H__ */ diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 5e56288e343b..a094f964c869 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -445,15 +445,10 @@ static inline void __sync_cache_range_r(volatile void *p, size_t size) * however some exceptions may exist. Caveat emptor. * * - The clobber list is dictated by the call to v7_flush_dcache_*. - * fp is preserved to the stack explicitly prior disabling the cache - * since adding it to the clobber list is incompatible with having - * CONFIG_FRAME_POINTER=y. ip is saved as well if ever r12-clobbering - * trampoline are inserted by the linker and to keep sp 64-bit aligned. */ #define v7_exit_coherency_flush(level) \ asm volatile( \ ".arch armv7-a \n\t" \ - "stmfd sp!, {fp, ip} \n\t" \ "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR \n\t" \ "bic r0, r0, #"__stringify(CR_C)" \n\t" \ "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR \n\t" \ @@ -463,10 +458,9 @@ static inline void __sync_cache_range_r(volatile void *p, size_t size) "bic r0, r0, #(1 << 6) @ disable local coherency \n\t" \ "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR \n\t" \ "isb \n\t" \ - "dsb \n\t" \ - "ldmfd sp!, {fp, ip}" \ - : : : "r0","r1","r2","r3","r4","r5","r6","r7", \ - "r9","r10","lr","memory" ) + "dsb" \ + : : : "r0","r1","r2","r3","r4","r5","r6", \ + "r9","r10","ip","lr","memory" ) void flush_uprobe_xol_access(struct page *page, unsigned long uaddr, void *kaddr, unsigned long len); diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 6bf0aad672c3..1e1178bf176d 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -8,25 +8,18 @@ #define _ASM_ARM_CURRENT_H #ifndef __ASSEMBLY__ +#include <asm/insn.h> struct task_struct; -static inline void set_current(struct task_struct *cur) -{ - if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) - return; - - /* Set TPIDRURO */ - asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); -} - -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO +extern struct task_struct *__current; -static inline struct task_struct *get_current(void) +static __always_inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; #if __has_builtin(__builtin_thread_pointer) && \ + defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && \ !(defined(CONFIG_THUMB2_KERNEL) && \ defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 130001) /* @@ -39,16 +32,39 @@ static inline struct task_struct *get_current(void) * https://github.com/ClangBuiltLinux/linux/issues/1485 */ cur = __builtin_thread_pointer(); +#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) + asm("0: mrc p15, 0, %0, c13, c0, 3 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" + " b 1b \n\t" #else - asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur)); + "2: ldr %0, 3f \n\t" + " ldr %0, [%0] \n\t" + " b 1b \n\t" + "3: .long __current \n\t" +#endif + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r"(cur)); +#elif __LINUX_ARM_ARCH__>= 7 || \ + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + cur = __current; +#else + asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); #endif return cur; } #define current get_current() -#else -#include <asm-generic/current.h> -#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h index b8102a6ddf16..d68101655b74 100644 --- a/arch/arm/include/asm/elf.h +++ b/arch/arm/include/asm/elf.h @@ -61,6 +61,9 @@ typedef struct user_fp elf_fpregset_t; #define R_ARM_MOVT_ABS 44 #define R_ARM_MOVW_PREL_NC 45 #define R_ARM_MOVT_PREL 46 +#define R_ARM_ALU_PC_G0_NC 57 +#define R_ARM_ALU_PC_G1_NC 59 +#define R_ARM_LDR_PC_G2 63 #define R_ARM_THM_CALL 10 #define R_ARM_THM_JUMP24 30 diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S deleted file mode 100644 index dfc6bfa43012..000000000000 --- a/arch/arm/include/asm/entry-macro-multi.S +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include <asm/assembler.h> - -/* - * Interrupt handling. Preserves r7, r8, r9 - */ - .macro arch_irq_handler_default - get_irqnr_preamble r6, lr -1: get_irqnr_and_base r0, r2, r6, lr - movne r1, sp - @ - @ routine called with r0 = irq number, r1 = struct pt_regs * - @ - badrne lr, 1b - bne asm_do_IRQ - -#ifdef CONFIG_SMP - /* - * XXX - * - * this macro assumes that irqstat (r2) and base (r6) are - * preserved from get_irqnr_and_base above - */ - ALT_SMP(test_for_ipi r0, r2, r6, lr) - ALT_UP_B(9997f) - movne r1, sp - badrne lr, 1b - bne do_IPI -#endif -9997: - .endm - - .macro arch_irq_handler, symbol_name - .align 5 - .global \symbol_name -\symbol_name: - mov r8, lr - arch_irq_handler_default - ret r8 - .endm diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index a4dbac07e4ef..7e9251ca29fe 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -2,6 +2,8 @@ #ifndef _ASM_ARM_FTRACE #define _ASM_ARM_FTRACE +#define HAVE_FUNCTION_GRAPH_FP_TEST + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #define ARCH_SUPPORTS_FTRACE_OPS 1 #endif @@ -48,7 +50,7 @@ void *return_address(unsigned int); static inline void *return_address(unsigned int level) { - return NULL; + return NULL; } #endif diff --git a/arch/arm/include/asm/hardware/entry-macro-iomd.S b/arch/arm/include/asm/hardware/entry-macro-iomd.S deleted file mode 100644 index f7692731e514..000000000000 --- a/arch/arm/include/asm/hardware/entry-macro-iomd.S +++ /dev/null @@ -1,131 +0,0 @@ -/* - * arch/arm/include/asm/hardware/entry-macro-iomd.S - * - * Low-level IRQ helper macros for IOC/IOMD based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - -/* IOC / IOMD based hardware */ -#include <asm/hardware/iomd.h> - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldrb \irqstat, [\base, #IOMD_IRQREQB] @ get high priority first - ldr \tmp, =irq_prio_h - teq \irqstat, #0 -#ifdef IOMD_BASE - ldrbeq \irqstat, [\base, #IOMD_DMAREQ] @ get dma - addeq \tmp, \tmp, #256 @ irq_prio_h table size - teqeq \irqstat, #0 - bne 2406f -#endif - ldrbeq \irqstat, [\base, #IOMD_IRQREQA] @ get low priority - addeq \tmp, \tmp, #256 @ irq_prio_d table size - teqeq \irqstat, #0 -#ifdef IOMD_IRQREQC - ldrbeq \irqstat, [\base, #IOMD_IRQREQC] - addeq \tmp, \tmp, #256 @ irq_prio_l table size - teqeq \irqstat, #0 -#endif -#ifdef IOMD_IRQREQD - ldrbeq \irqstat, [\base, #IOMD_IRQREQD] - addeq \tmp, \tmp, #256 @ irq_prio_lc table size - teqeq \irqstat, #0 -#endif -2406: ldrbne \irqnr, [\tmp, \irqstat] @ get IRQ number - .endm - -/* - * Interrupt table (incorporates priority). Please note that we - * rely on the order of these tables (see above code). - */ - .align 5 -irq_prio_h: .byte 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 -#ifdef IOMD_BASE -irq_prio_d: .byte 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 -#endif -irq_prio_l: .byte 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 -#ifdef IOMD_IRQREQC -irq_prio_lc: .byte 24,24,25,24,26,26,26,26,27,27,27,27,27,27,27,27 - .byte 28,24,25,24,26,26,26,26,27,27,27,27,27,27,27,27 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 30,30,30,30,30,30,30,30,27,27,27,27,27,27,27,27 - .byte 30,30,30,30,30,30,30,30,27,27,27,27,27,27,27,27 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 -#endif -#ifdef IOMD_IRQREQD -irq_prio_ld: .byte 40,40,41,40,42,42,42,42,43,43,43,43,43,43,43,43 - .byte 44,40,41,40,42,42,42,42,43,43,43,43,43,43,43,43 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 46,46,46,46,46,46,46,46,43,43,43,43,43,43,43,43 - .byte 46,46,46,46,46,46,46,46,43,43,43,43,43,43,43,43 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 -#endif - diff --git a/arch/arm/include/asm/insn.h b/arch/arm/include/asm/insn.h index 5475cbf9fb6b..faf3d1c28368 100644 --- a/arch/arm/include/asm/insn.h +++ b/arch/arm/include/asm/insn.h @@ -2,6 +2,23 @@ #ifndef __ASM_ARM_INSN_H #define __ASM_ARM_INSN_H +#include <linux/types.h> + +/* + * Avoid a literal load by emitting a sequence of ADD/LDR instructions with the + * appropriate relocations. The combined sequence has a range of -/+ 256 MiB, + * which should be sufficient for the core kernel as well as modules loaded + * into the module region. (Not supported by LLD before release 14) + */ +#define LOAD_SYM_ARMV6(reg, sym) \ + " .globl " #sym " \n\t" \ + " .reloc 10f, R_ARM_ALU_PC_G0_NC, " #sym " \n\t" \ + " .reloc 11f, R_ARM_ALU_PC_G1_NC, " #sym " \n\t" \ + " .reloc 12f, R_ARM_LDR_PC_G2, " #sym " \n\t" \ + "10: sub " #reg ", pc, #8 \n\t" \ + "11: sub " #reg ", " #reg ", #4 \n\t" \ + "12: ldr " #reg ", [" #reg ", #0] \n\t" + static inline unsigned long arm_gen_nop(void) { diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 1cbcc462b07e..a7c2337b0c7d 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -26,7 +26,6 @@ struct irqaction; struct pt_regs; -extern void asm_do_IRQ(unsigned int, struct pt_regs *); void handle_IRQ(unsigned int, struct pt_regs *); void init_IRQ(void); diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index eec0c0bda766..9349e7a82c9c 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -56,9 +56,7 @@ struct machine_desc { void (*init_time)(void); void (*init_machine)(void); void (*init_late)(void); -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER void (*handle_irq)(struct pt_regs *); -#endif void (*restart)(enum reboot_mode, const char *); }; diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index 1592a4264488..e049723840d3 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -10,7 +10,7 @@ typedef struct { #else int switch_pending; #endif - unsigned int vmalloc_seq; + atomic_t vmalloc_seq; unsigned long sigpage; #ifdef CONFIG_VDSO unsigned long vdso; diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 84e58956fcab..db2cb06aa8cf 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -23,6 +23,16 @@ void __check_vmalloc_seq(struct mm_struct *mm); +#ifdef CONFIG_MMU +static inline void check_vmalloc_seq(struct mm_struct *mm) +{ + if (!IS_ENABLED(CONFIG_ARM_LPAE) && + unlikely(atomic_read(&mm->context.vmalloc_seq) != + atomic_read(&init_mm.context.vmalloc_seq))) + __check_vmalloc_seq(mm); +} +#endif + #ifdef CONFIG_CPU_HAS_ASID void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk); @@ -52,8 +62,7 @@ static inline void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm, static inline void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) { - if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) - __check_vmalloc_seq(mm); + check_vmalloc_seq(mm); if (irqs_disabled()) /* @@ -129,6 +138,15 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, #endif } +#ifdef CONFIG_VMAP_STACK +static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +{ + if (mm != &init_mm) + check_vmalloc_seq(mm); +} +#define enter_lazy_tlb enter_lazy_tlb +#endif + #include <asm-generic/mmu_context.h> #endif diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 11b058a72a5b..5fcc8a600e36 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -147,6 +147,9 @@ extern void copy_page(void *to, const void *from); #include <asm/pgtable-3level-types.h> #else #include <asm/pgtable-2level-types.h> +#ifdef CONFIG_VMAP_STACK +#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED +#endif #endif #endif /* CONFIG_MMU */ diff --git a/arch/arm/include/asm/paravirt_api_clock.h b/arch/arm/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..65ac7cee0dad --- /dev/null +++ b/arch/arm/include/asm/paravirt_api_clock.h @@ -0,0 +1 @@ +#include <asm/paravirt.h> diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index e2fcb3cfd3de..7545c87c251f 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -5,20 +5,27 @@ #ifndef _ASM_ARM_PERCPU_H_ #define _ASM_ARM_PERCPU_H_ +#include <asm/insn.h> + register unsigned long current_stack_pointer asm ("sp"); /* * Same as asm-generic/percpu.h, except that we store the per cpu offset * in the TPIDRPRW. TPIDRPRW only exists on V6K and V7 */ -#if defined(CONFIG_SMP) && !defined(CONFIG_CPU_V6) +#ifdef CONFIG_SMP static inline void set_my_cpu_offset(unsigned long off) { + extern unsigned int smp_on_up; + + if (IS_ENABLED(CONFIG_CPU_V6) && !smp_on_up) + return; + /* Set TPIDRPRW */ asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory"); } -static inline unsigned long __my_cpu_offset(void) +static __always_inline unsigned long __my_cpu_offset(void) { unsigned long off; @@ -27,8 +34,28 @@ static inline unsigned long __my_cpu_offset(void) * We want to allow caching the value, so avoid using volatile and * instead use a fake stack read to hazard against barrier(). */ - asm("mrc p15, 0, %0, c13, c0, 4" : "=r" (off) - : "Q" (*(const unsigned long *)current_stack_pointer)); + asm("0: mrc p15, 0, %0, c13, c0, 4 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" + " b 1b \n\t" +#else + "2: ldr %0, 3f \n\t" + " ldr %0, [%0] \n\t" + " b 1b \n\t" + "3: .long __per_cpu_offset \n\t" +#endif + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r" (off) + : "Q" (*(const unsigned long *)current_stack_pointer)); return off; } diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index 70fe69bdcce2..92abd4cd8ca2 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -208,6 +208,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) } #define pmd_offset pmd_offset +#define pmd_pfn(pmd) (__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) + #define pmd_large(pmd) (pmd_val(pmd) & 2) #define pmd_leaf(pmd) (pmd_val(pmd) & 2) #define pmd_bad(pmd) (pmd_val(pmd) & 2) diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index f16cbbd5cda4..7c1c90d9f582 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -25,11 +25,6 @@ struct seq_file; extern void show_ipi_list(struct seq_file *, int); /* - * Called from assembly code, this handles an IPI. - */ -asmlinkage void do_IPI(int ipinr, struct pt_regs *regs); - -/* * Called from C code, this handles an IPI. */ void handle_IPI(int ipinr, struct pt_regs *regs); diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h new file mode 100644 index 000000000000..85f9e538fb32 --- /dev/null +++ b/arch/arm/include/asm/spectre.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_SPECTRE_H +#define __ASM_SPECTRE_H + +enum { + SPECTRE_UNAFFECTED, + SPECTRE_MITIGATED, + SPECTRE_VULNERABLE, +}; + +enum { + __SPECTRE_V2_METHOD_BPIALL, + __SPECTRE_V2_METHOD_ICIALLU, + __SPECTRE_V2_METHOD_SMC, + __SPECTRE_V2_METHOD_HVC, + __SPECTRE_V2_METHOD_LOOP8, +}; + +enum { + SPECTRE_V2_METHOD_BPIALL = BIT(__SPECTRE_V2_METHOD_BPIALL), + SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), + SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), + SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), + SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), +}; + +#ifdef CONFIG_GENERIC_CPU_VULNERABILITIES +void spectre_v2_update_state(unsigned int state, unsigned int methods); +#else +static inline void spectre_v2_update_state(unsigned int state, + unsigned int methods) +{} +#endif + +int spectre_bhb_update_vectors(unsigned int method); + +#endif diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 8f54f9ad8a9b..3e78f921b8b2 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -14,6 +14,9 @@ struct stackframe { unsigned long sp; unsigned long lr; unsigned long pc; + + /* address of the LR value on the stack */ + unsigned long *lr_addr; #ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; struct task_struct *tsk; @@ -36,5 +39,7 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) extern int unwind_frame(struct stackframe *frame); extern void walk_stackframe(struct stackframe *frame, int (*fn)(struct stackframe *, void *), void *data); +extern void dump_mem(const char *lvl, const char *str, unsigned long bottom, + unsigned long top); #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index 61e4a3c4ca6e..9372348516ce 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -3,6 +3,7 @@ #define __ASM_ARM_SWITCH_TO_H #include <linux/thread_info.h> +#include <asm/smp_plat.h> /* * For v7 SMP cores running a preemptible kernel we may be pre-empted @@ -26,7 +27,7 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info #define switch_to(prev,next,last) \ do { \ __complete_pending_tlbi(); \ - if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \ + if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || is_smp()) \ __this_cpu_write(__entry_task, next); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 164e15f26485..aecc403b2880 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -25,6 +25,14 @@ #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define THREAD_START_SP (THREAD_SIZE - 8) +#ifdef CONFIG_VMAP_STACK +#define THREAD_ALIGN (2 * THREAD_SIZE) +#else +#define THREAD_ALIGN THREAD_SIZE +#endif + +#define OVERFLOW_STACK_SIZE SZ_4K + #ifndef __ASSEMBLY__ struct task_struct; @@ -54,9 +62,6 @@ struct cpu_context_save { struct thread_info { unsigned long flags; /* low level flags */ int preempt_count; /* 0 => preemptable, <0 => bug */ -#ifndef CONFIG_THREAD_INFO_IN_TASK - struct task_struct *task; /* main task structure */ -#endif __u32 cpu; /* cpu */ __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ @@ -72,39 +77,15 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ - INIT_THREAD_INFO_TASK(tsk) \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ } -#ifdef CONFIG_THREAD_INFO_IN_TASK -#define INIT_THREAD_INFO_TASK(tsk) - static inline struct task_struct *thread_task(struct thread_info* ti) { return (struct task_struct *)ti; } -#else -#define INIT_THREAD_INFO_TASK(tsk) .task = &(tsk), - -static inline struct task_struct *thread_task(struct thread_info* ti) -{ - return ti->task; -} - -/* - * how to get the thread information struct from C - */ -static inline struct thread_info *current_thread_info(void) __attribute_const__; - -static inline struct thread_info *current_thread_info(void) -{ - return (struct thread_info *) - (current_stack_pointer & ~(THREAD_SIZE - 1)); -} -#endif - #define thread_saved_pc(tsk) \ ((unsigned long)(task_thread_info(tsk)->cpu_context.pc)) #define thread_saved_sp(tsk) \ diff --git a/arch/arm/include/asm/tls.h b/arch/arm/include/asm/tls.h index c3296499176c..3dcd0f71a0da 100644 --- a/arch/arm/include/asm/tls.h +++ b/arch/arm/include/asm/tls.h @@ -18,21 +18,32 @@ .endm .macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2 - ldr \tmp1, =elf_hwcap - ldr \tmp1, [\tmp1, #0] +#ifdef CONFIG_SMP +ALT_SMP(nop) +ALT_UP_B(.L0_\@) + .subsection 1 +#endif +.L0_\@: + ldr_va \tmp1, elf_hwcap mov \tmp2, #0xffff0fff tst \tmp1, #HWCAP_TLS @ hardware TLS available? streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0 - mrcne p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register - mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register - mcrne p15, 0, \tpuser, c13, c0, 2 @ set user r/w register - strne \tmp2, [\base, #TI_TP_VALUE + 4] @ save it + beq .L2_\@ + mcr p15, 0, \tp, c13, c0, 3 @ yes, set TLS register +#ifdef CONFIG_SMP + b .L1_\@ + .previous +#endif +.L1_\@: switch_tls_v6k \base, \tp, \tpuser, \tmp1, \tmp2 +.L2_\@: .endm .macro switch_tls_software, base, tp, tpuser, tmp1, tmp2 mov \tmp1, #0xffff0fff str \tp, [\tmp1, #-15] @ set TLS value at 0xffff0ff0 .endm +#else +#include <asm/smp_plat.h> #endif #ifdef CONFIG_TLS_REG_EMUL @@ -43,7 +54,7 @@ #elif defined(CONFIG_CPU_V6) #define tls_emu 0 #define has_tls_reg (elf_hwcap & HWCAP_TLS) -#define defer_tls_reg_update 0 +#define defer_tls_reg_update is_smp() #define switch_tls switch_tls_v6 #elif defined(CONFIG_CPU_32v6K) #define tls_emu 0 @@ -81,11 +92,11 @@ static inline void set_tls(unsigned long val) */ barrier(); - if (!tls_emu && !defer_tls_reg_update) { - if (has_tls_reg) { + if (!tls_emu) { + if (has_tls_reg && !defer_tls_reg_update) { asm("mcr p15, 0, %0, c13, c0, 3" : : "r" (val)); - } else { + } else if (!has_tls_reg) { #ifdef CONFIG_KUSER_HELPERS /* * User space must never try to access this diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 32dbfd81f42a..2fcbec9c306c 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -56,21 +56,6 @@ extern int __put_user_bad(void); #ifdef CONFIG_MMU /* - * We use 33-bit arithmetic here. Success returns zero, failure returns - * addr_limit. We take advantage that addr_limit will be zero for KERNEL_DS, - * so this will always return success in that case. - */ -#define __range_ok(addr, size) ({ \ - unsigned long flag, roksum; \ - __chk_user_ptr(addr); \ - __asm__(".syntax unified\n" \ - "adds %1, %2, %3; sbcscc %1, %1, %0; movcc %0, #0" \ - : "=&r" (flag), "=&r" (roksum) \ - : "r" (addr), "Ir" (size), "0" (TASK_SIZE) \ - : "cc"); \ - flag; }) - -/* * This is a type: either unsigned long, if the argument fits into * that type, or otherwise unsigned long long. */ @@ -241,15 +226,12 @@ extern int __put_user_8(void *, unsigned long long); #else /* CONFIG_MMU */ -#define __addr_ok(addr) ((void)(addr), 1) -#define __range_ok(addr, size) ((void)(addr), 0) - #define get_user(x, p) __get_user(x, p) #define __put_user_check __put_user_nocheck #endif /* CONFIG_MMU */ -#define access_ok(addr, size) (__range_ok(addr, size) == 0) +#include <asm-generic/access_ok.h> #ifdef CONFIG_CPU_SPECTRE /* @@ -476,8 +458,6 @@ do { \ : "r" (x), "i" (-EFAULT) \ : "cc") -#define HAVE_GET_KERNEL_NOFAULT - #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ const type *__pk_ptr = (src); \ diff --git a/arch/arm/include/asm/v7m.h b/arch/arm/include/asm/v7m.h index 2cb00d15831b..4512f7e1918f 100644 --- a/arch/arm/include/asm/v7m.h +++ b/arch/arm/include/asm/v7m.h @@ -13,6 +13,7 @@ #define V7M_SCB_ICSR_PENDSVSET (1 << 28) #define V7M_SCB_ICSR_PENDSVCLR (1 << 27) #define V7M_SCB_ICSR_RETTOBASE (1 << 11) +#define V7M_SCB_ICSR_VECTACTIVE 0x000001ff #define V7M_SCB_VTOR 0x08 @@ -38,7 +39,7 @@ #define V7M_SCB_SHCSR_MEMFAULTENA (1 << 16) #define V7M_xPSR_FRAMEPTRALIGN 0x00000200 -#define V7M_xPSR_EXCEPTIONNO 0x000001ff +#define V7M_xPSR_EXCEPTIONNO V7M_SCB_ICSR_VECTACTIVE /* * When branching to an address that has bits [31:28] == 0xf an exception return diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index 4a91428c324d..fad45c884e98 100644 --- a/arch/arm/include/asm/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -26,6 +26,19 @@ #define ARM_MMU_DISCARD(x) x #endif +/* + * ld.lld does not support NOCROSSREFS: + * https://github.com/ClangBuiltLinux/linux/issues/1609 + */ +#ifdef CONFIG_LD_IS_LLD +#define NOCROSSREFS +#endif + +/* Set start/end symbol names to the LMA for the section */ +#define ARM_LMA(sym, section) \ + sym##_start = LOADADDR(section); \ + sym##_end = LOADADDR(section) + SIZEOF(section) + #define PROC_INFO \ . = ALIGN(4); \ __proc_info_begin = .; \ @@ -110,19 +123,31 @@ * only thing that matters is their relative offsets */ #define ARM_VECTORS \ - __vectors_start = .; \ - .vectors 0xffff0000 : AT(__vectors_start) { \ - *(.vectors) \ + __vectors_lma = .; \ + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { \ + .vectors { \ + *(.vectors) \ + } \ + .vectors.bhb.loop8 { \ + *(.vectors.bhb.loop8) \ + } \ + .vectors.bhb.bpiall { \ + *(.vectors.bhb.bpiall) \ + } \ } \ - . = __vectors_start + SIZEOF(.vectors); \ - __vectors_end = .; \ + ARM_LMA(__vectors, .vectors); \ + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); \ + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); \ + . = __vectors_lma + SIZEOF(.vectors) + \ + SIZEOF(.vectors.bhb.loop8) + \ + SIZEOF(.vectors.bhb.bpiall); \ \ - __stubs_start = .; \ - .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \ + __stubs_lma = .; \ + .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { \ *(.stubs) \ } \ - . = __stubs_start + SIZEOF(.stubs); \ - __stubs_end = .; \ + ARM_LMA(__stubs, .stubs); \ + . = __stubs_lma + SIZEOF(.stubs); \ \ PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors)); diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h index aefddec79286..669cad5194d3 100644 --- a/arch/arm/include/asm/xor.h +++ b/arch/arm/include/asm/xor.h @@ -44,7 +44,8 @@ : "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4)) static void -xor_arm4regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned int lines = bytes / sizeof(unsigned long) / 4; register unsigned int a1 __asm__("r4"); @@ -64,8 +65,9 @@ xor_arm4regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_arm4regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned int lines = bytes / sizeof(unsigned long) / 4; register unsigned int a1 __asm__("r4"); @@ -86,8 +88,10 @@ xor_arm4regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_arm4regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned int lines = bytes / sizeof(unsigned long) / 2; register unsigned int a1 __asm__("r8"); @@ -105,8 +109,11 @@ xor_arm4regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_arm4regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { unsigned int lines = bytes / sizeof(unsigned long) / 2; register unsigned int a1 __asm__("r8"); @@ -146,7 +153,8 @@ static struct xor_block_template xor_block_arm4regs = { extern struct xor_block_template const xor_block_neon_inner; static void -xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { if (in_interrupt()) { xor_arm4regs_2(bytes, p1, p2); @@ -158,8 +166,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { if (in_interrupt()) { xor_arm4regs_3(bytes, p1, p2, p3); @@ -171,8 +180,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { if (in_interrupt()) { xor_arm4regs_4(bytes, p1, p2, p3, p4); @@ -184,8 +195,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { if (in_interrupt()) { xor_arm4regs_5(bytes, p1, p2, p3, p4, p5); diff --git a/arch/arm/include/uapi/asm/signal.h b/arch/arm/include/uapi/asm/signal.h index c9a3ea1d8d41..9e2178420db2 100644 --- a/arch/arm/include/uapi/asm/signal.h +++ b/arch/arm/include/uapi/asm/signal.h @@ -93,7 +93,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index ae295a3bcfef..553866751e1a 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -10,6 +10,7 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_patch.o = -pg +CFLAGS_REMOVE_unwind.o = -pg endif CFLAGS_REMOVE_return_address.o = -pg @@ -106,4 +107,6 @@ endif obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o +obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o + extra-y := $(head-y) vmlinux.lds diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 645845e4982a..2c8d76fd7c66 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -43,9 +43,6 @@ int main(void) BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); -#ifndef CONFIG_THREAD_INFO_IN_TASK - DEFINE(TI_TASK, offsetof(struct thread_info, task)); -#endif DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 5cd057859fe9..06508698abb8 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -19,9 +19,6 @@ #include <asm/glue-df.h> #include <asm/glue-pf.h> #include <asm/vfpmacros.h> -#ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER -#include <mach/entry-macro.S> -#endif #include <asm/thread_notify.h> #include <asm/unwind.h> #include <asm/unistd.h> @@ -30,19 +27,35 @@ #include <asm/uaccess-asm.h> #include "entry-header.S" -#include <asm/entry-macro-multi.S> #include <asm/probes.h> /* * Interrupt handling. */ - .macro irq_handler -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER - mov r0, sp - bl generic_handle_arch_irq -#else - arch_irq_handler_default + .macro irq_handler, from_user:req + mov r1, sp + ldr_this_cpu r2, irq_stack_ptr, r2, r3 + .if \from_user == 0 + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r3, r2, r1 @ SP above bottom of IRQ stack? + rsbscs r3, r3, #THREAD_SIZE @ ... and below the top? +#ifdef CONFIG_VMAP_STACK + ldr_va r3, high_memory, cc @ End of the linear region + cmpcc r3, r1 @ Stack pointer was below it? #endif + bcc 0f @ If not, switch to the IRQ stack + mov r0, r1 + bl generic_handle_arch_irq + b 1f +0: + .endif + + mov_l r0, generic_handle_arch_irq + bl call_with_stack +1: .endm .macro pabt_helper @@ -140,27 +153,35 @@ ENDPROC(__und_invalid) #define SPFIX(code...) #endif - .macro svc_entry, stack_hole=0, trace=1, uaccess=1 + .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1 UNWIND(.fnstart ) - UNWIND(.save {r0 - pc} ) - sub sp, sp, #(SVC_REGS_SIZE + \stack_hole - 4) + sub sp, sp, #(SVC_REGS_SIZE + \stack_hole) + THUMB( add sp, r1 ) @ get SP in a GPR without + THUMB( sub r1, sp, r1 ) @ using a temp register + + .if \overflow_check + UNWIND(.save {r0 - pc} ) + do_overflow_check (SVC_REGS_SIZE + \stack_hole) + .endif + #ifdef CONFIG_THUMB2_KERNEL - SPFIX( str r0, [sp] ) @ temporarily saved - SPFIX( mov r0, sp ) - SPFIX( tst r0, #4 ) @ test original stack alignment - SPFIX( ldr r0, [sp] ) @ restored + tst r1, #4 @ test stack pointer alignment + sub r1, sp, r1 @ restore original R1 + sub sp, r1 @ restore original SP #else SPFIX( tst sp, #4 ) #endif - SPFIX( subeq sp, sp, #4 ) - stmia sp, {r1 - r12} + SPFIX( subne sp, sp, #4 ) + + ARM( stmib sp, {r1 - r12} ) + THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2 ldmia r0, {r3 - r5} - add r7, sp, #S_SP - 4 @ here for interlock avoidance + add r7, sp, #S_SP @ here for interlock avoidance mov r6, #-1 @ "" "" "" "" - add r2, sp, #(SVC_REGS_SIZE + \stack_hole - 4) - SPFIX( addeq r2, r2, #4 ) - str r3, [sp, #-4]! @ save the "real" r0 copied + add r2, sp, #(SVC_REGS_SIZE + \stack_hole) + SPFIX( addne r2, r2, #4 ) + str r3, [sp] @ save the "real" r0 copied @ from the exception stack mov r3, lr @@ -199,7 +220,7 @@ ENDPROC(__dabt_svc) .align 5 __irq_svc: svc_entry - irq_handler + irq_handler from_user=0 #ifdef CONFIG_PREEMPTION ldr r8, [tsk, #TI_PREEMPT] @ get preempt count @@ -426,7 +447,7 @@ ENDPROC(__dabt_usr) __irq_usr: usr_entry kuser_cmpxchg_check - irq_handler + irq_handler from_user=1 get_thread_info tsk mov why, #0 b ret_to_user_from_irq @@ -752,16 +773,17 @@ ENTRY(__switch_to) ldr r6, [r2, #TI_CPU_DOMAIN] #endif switch_tls r1, r4, r5, r3, r7 -#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) - ldr r7, [r2, #TI_TASK] +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ + !defined(CONFIG_STACKPROTECTOR_PER_TASK) ldr r8, =__stack_chk_guard .if (TSK_STACK_CANARY > IMM12_MASK) - add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK + add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK + ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK] + .else + ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK] .endif - ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK] -#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) - mov r7, r2 @ Preserve 'next' #endif + mov r7, r2 @ Preserve 'next' #ifdef CONFIG_CPU_USE_DOMAINS mcr p15, 0, r6, c3, c0, 0 @ Set domain register #endif @@ -770,19 +792,102 @@ ENTRY(__switch_to) ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain -#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) - str r7, [r8] +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ + !defined(CONFIG_STACKPROTECTOR_PER_TASK) + str r9, [r8] #endif - THUMB( mov ip, r4 ) mov r0, r5 - set_current r7 - ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously - THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously - THUMB( ldr sp, [ip], #4 ) - THUMB( ldr pc, [ip] ) +#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) + set_current r7, r8 + ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously +#else + mov r1, r7 + ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously +#ifdef CONFIG_VMAP_STACK + @ + @ Do a dummy read from the new stack while running from the old one so + @ that we can rely on do_translation_fault() to fix up any stale PMD + @ entries covering the vmalloc region. + @ + ldr r2, [ip] +#endif + + @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what + @ effectuates the task switch, as that is what causes the observable + @ values of current and current_thread_info to change. When + @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore + @ current_thread_info) is done explicitly, and the update of SP just + @ switches us to another stack, with few other side effects. In order + @ to prevent this distinction from causing any inconsistencies, let's + @ keep the 'set_current' call as close as we can to the update of SP. + set_current r1, r2 + mov sp, ip + ret lr +#endif UNWIND(.fnend ) ENDPROC(__switch_to) +#ifdef CONFIG_VMAP_STACK + .text + .align 2 +__bad_stack: + @ + @ We've just detected an overflow. We need to load the address of this + @ CPU's overflow stack into the stack pointer register. We have only one + @ scratch register so let's use a sequence of ADDs including one + @ involving the PC, and decorate them with PC-relative group + @ relocations. As these are ARM only, switch to ARM mode first. + @ + @ We enter here with IP clobbered and its value stashed on the mode + @ stack. + @ +THUMB( bx pc ) +THUMB( nop ) +THUMB( .arm ) + ldr_this_cpu_armv6 ip, overflow_stack_ptr + + str sp, [ip, #-4]! @ Preserve original SP value + mov sp, ip @ Switch to overflow stack + pop {ip} @ Original SP in IP + +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + mov ip, ip @ mov expected by unwinder + push {fp, ip, lr, pc} @ GCC flavor frame record +#else + str ip, [sp, #-8]! @ store original SP + push {fpreg, lr} @ Clang flavor frame record +#endif +UNWIND( ldr ip, [r0, #4] ) @ load exception LR +UNWIND( str ip, [sp, #12] ) @ store in the frame record + ldr ip, [r0, #12] @ reload IP + + @ Store the original GPRs to the new stack. + svc_entry uaccess=0, overflow_check=0 + +UNWIND( .save {sp, pc} ) +UNWIND( .save {fpreg, lr} ) +UNWIND( .setfp fpreg, sp ) + + ldr fpreg, [sp, #S_SP] @ Add our frame record + @ to the linked list +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + ldr r1, [fp, #4] @ reload SP at entry + add fp, fp, #12 +#else + ldr r1, [fpreg, #8] +#endif + str r1, [sp, #S_SP] @ store in pt_regs + + @ Stash the regs for handle_bad_stack + mov r0, sp + + @ Time to die + bl handle_bad_stack + nop +UNWIND( .fnend ) +ENDPROC(__bad_stack) +#endif + __INIT /* @@ -1002,12 +1107,11 @@ vector_\name: sub lr, lr, #\correction .endif - @ - @ Save r0, lr_<exception> (parent PC) and spsr_<exception> - @ (parent CPSR) - @ + @ Save r0, lr_<exception> (parent PC) stmia sp, {r0, lr} @ save r0, lr - mrs lr, spsr + + @ Save spsr_<exception> (parent CPSR) +2: mrs lr, spsr str lr, [sp, #8] @ save spsr @ @@ -1028,6 +1132,44 @@ vector_\name: movs pc, lr @ branch to handler in SVC mode ENDPROC(vector_\name) +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .subsection 1 + .align 5 +vector_bhb_loop8_\name: + .if \correction + sub lr, lr, #\correction + .endif + + @ Save r0, lr_<exception> (parent PC) + stmia sp, {r0, lr} + + @ bhb workaround + mov r0, #8 +3: b . + 4 + subs r0, r0, #1 + bne 3b + dsb + isb + b 2b +ENDPROC(vector_bhb_loop8_\name) + +vector_bhb_bpiall_\name: + .if \correction + sub lr, lr, #\correction + .endif + + @ Save r0, lr_<exception> (parent PC) + stmia sp, {r0, lr} + + @ bhb workaround + mcr p15, 0, r0, c7, c5, 6 @ BPIALL + @ isb not needed due to "movs pc, lr" in the vector stub + @ which gives a "context synchronisation". + b 2b +ENDPROC(vector_bhb_bpiall_\name) + .previous +#endif + .align 2 @ handler addresses follow this label 1: @@ -1036,6 +1178,10 @@ ENDPROC(vector_\name) .section .stubs, "ax", %progbits @ This must be the first word .word vector_swi +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .word vector_bhb_loop8_swi + .word vector_bhb_bpiall_swi +#endif vector_rst: ARM( swi SYS_ERROR0 ) @@ -1150,8 +1296,10 @@ vector_addrexcptn: * FIQ "NMI" handler *----------------------------------------------------------------------------- * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 - * systems. + * systems. This must be the last vector stub, so lets place it in its own + * subsection. */ + .subsection 2 vector_stub fiq, FIQ_MODE, 4 .long __fiq_usr @ 0 (USR_26 / USR_32) @@ -1184,6 +1332,30 @@ vector_addrexcptn: W(b) vector_irq W(b) vector_fiq +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .section .vectors.bhb.loop8, "ax", %progbits +.L__vectors_bhb_loop8_start: + W(b) vector_rst + W(b) vector_bhb_loop8_und + W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004 + W(b) vector_bhb_loop8_pabt + W(b) vector_bhb_loop8_dabt + W(b) vector_addrexcptn + W(b) vector_bhb_loop8_irq + W(b) vector_bhb_loop8_fiq + + .section .vectors.bhb.bpiall, "ax", %progbits +.L__vectors_bhb_bpiall_start: + W(b) vector_rst + W(b) vector_bhb_bpiall_und + W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008 + W(b) vector_bhb_bpiall_pabt + W(b) vector_bhb_bpiall_dabt + W(b) vector_addrexcptn + W(b) vector_bhb_bpiall_irq + W(b) vector_bhb_bpiall_fiq +#endif + .data .align 2 diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index ac86c34682bb..90d40f4d56cf 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -16,12 +16,14 @@ .equ NR_syscalls, __NR_syscalls -#ifdef CONFIG_NEED_RET_TO_USER -#include <mach/entry-macro.S> -#else - .macro arch_ret_to_user, tmp1, tmp2 - .endm + .macro arch_ret_to_user, tmp +#ifdef CONFIG_ARCH_IOP32X + mrc p15, 0, \tmp, c15, c1, 0 + tst \tmp, #(1 << 6) + bicne \tmp, \tmp, #(1 << 6) + mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access #endif + .endm #include "entry-header.S" @@ -55,7 +57,7 @@ __ret_fast_syscall: /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) @@ -128,7 +130,7 @@ no_work_pending: asm_trace_hardirqs_on save = 0 /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 ct_user_enter save = 0 restore_user_regs fast = 0, offset = 0 @@ -154,12 +156,36 @@ ENDPROC(ret_from_fork) */ .align 5 +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +ENTRY(vector_bhb_loop8_swi) + sub sp, sp, #PT_REGS_SIZE + stmia sp, {r0 - r12} + mov r8, #8 +1: b 2f +2: subs r8, r8, #1 + bne 1b + dsb + isb + b 3f +ENDPROC(vector_bhb_loop8_swi) + + .align 5 +ENTRY(vector_bhb_bpiall_swi) + sub sp, sp, #PT_REGS_SIZE + stmia sp, {r0 - r12} + mcr p15, 0, r8, c7, c5, 6 @ BPIALL + isb + b 3f +ENDPROC(vector_bhb_bpiall_swi) +#endif + .align 5 ENTRY(vector_swi) #ifdef CONFIG_CPU_V7M v7m_exception_entry #else sub sp, sp, #PT_REGS_SIZE stmia sp, {r0 - r12} @ Calling r0 - r12 +3: ARM( add r8, sp, #S_PC ) ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr THUMB( mov r8, sp ) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index a74289ebc803..3e7bcaca5e07 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -22,12 +22,9 @@ * mcount can be thought of as a function called in the middle of a subroutine * call. As such, it needs to be transparent for both the caller and the * callee: the original lr needs to be restored when leaving mcount, and no - * registers should be clobbered. (In the __gnu_mcount_nc implementation, we - * clobber the ip register. This is OK because the ARM calling convention - * allows it to be clobbered in subroutines and doesn't use it to hold - * parameters.) + * registers should be clobbered. * - * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}" + * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). */ @@ -38,23 +35,20 @@ .macro __mcount suffix mcount_enter - ldr r0, =ftrace_trace_function - ldr r2, [r0] - adr r0, .Lftrace_stub + ldr_va r2, ftrace_trace_function + badr r0, .Lftrace_stub cmp r0, r2 bne 1f #ifdef CONFIG_FUNCTION_GRAPH_TRACER - ldr r1, =ftrace_graph_return - ldr r2, [r1] - cmp r0, r2 - bne ftrace_graph_caller\suffix - - ldr r1, =ftrace_graph_entry - ldr r2, [r1] - ldr r0, =ftrace_graph_entry_stub - cmp r0, r2 - bne ftrace_graph_caller\suffix + ldr_va r2, ftrace_graph_return + cmp r0, r2 + bne ftrace_graph_caller\suffix + + ldr_va r2, ftrace_graph_entry + mov_l r0, ftrace_graph_entry_stub + cmp r0, r2 + bne ftrace_graph_caller\suffix #endif mcount_exit @@ -70,29 +64,27 @@ .macro __ftrace_regs_caller - sub sp, sp, #8 @ space for PC and CPSR OLD_R0, + str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0, @ OLD_R0 will overwrite previous LR - add ip, sp, #12 @ move in IP the value of SP as it was - @ before the push {lr} of the mcount mechanism + ldr lr, [sp, #8] @ get previous LR - str lr, [sp, #0] @ store LR instead of PC + str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR - ldr lr, [sp, #8] @ get previous LR + str lr, [sp, #-4]! @ store previous LR as LR - str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR + add lr, sp, #16 @ move in LR the value of SP as it was + @ before the push {lr} of the mcount mechanism - stmdb sp!, {ip, lr} - stmdb sp!, {r0-r11, lr} + push {r0-r11, ip, lr} @ stack content at this point: @ 0 4 48 52 56 60 64 68 72 - @ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 | + @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 | - mov r3, sp @ struct pt_regs* + mov r3, sp @ struct pt_regs* - ldr r2, =function_trace_op - ldr r2, [r2] @ pointer to the current + ldr_va r2, function_trace_op @ pointer to the current @ function tracing op ldr r1, [sp, #S_LR] @ lr of instrumented func @@ -108,35 +100,37 @@ ftrace_regs_call: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_regs_call ftrace_graph_regs_call: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif @ pop saved regs - ldmia sp!, {r0-r12} @ restore r0 through r12 - ldr ip, [sp, #8] @ restore PC - ldr lr, [sp, #4] @ restore LR - ldr sp, [sp, #0] @ restore SP - mov pc, ip @ return + pop {r0-r11, ip, lr} @ restore r0 through r12 + ldr lr, [sp], #4 @ restore LR + ldr pc, [sp], #12 .endm #ifdef CONFIG_FUNCTION_GRAPH_TRACER .macro __ftrace_graph_regs_caller - sub r0, fp, #4 @ lr of instrumented routine (parent) +#ifdef CONFIG_UNWINDER_FRAME_POINTER + sub r0, fp, #4 @ lr of instrumented routine (parent) +#else + add r0, sp, #S_LR +#endif @ called from __ftrace_regs_caller - ldr r1, [sp, #S_PC] @ instrumented routine (func) + ldr r1, [sp, #S_PC] @ instrumented routine (func) mcount_adjust_addr r1, r1 - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #PT_REGS_SIZE bl prepare_ftrace_return @ pop registers saved in ftrace_regs_caller - ldmia sp!, {r0-r12} @ restore r0 through r12 - ldr ip, [sp, #8] @ restore PC - ldr lr, [sp, #4] @ restore LR - ldr sp, [sp, #0] @ restore SP - mov pc, ip @ return + pop {r0-r11, ip, lr} @ restore r0 through r12 + ldr lr, [sp], #4 @ restore LR + ldr pc, [sp], #12 .endm #endif @@ -149,8 +143,7 @@ ftrace_graph_regs_call: mcount_adjust_addr r0, lr @ instrumented function #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS - ldr r2, =function_trace_op - ldr r2, [r2] @ pointer to the current + ldr_va r2, function_trace_op @ pointer to the current @ function tracing op mov r3, #0 @ regs is NULL #endif @@ -162,14 +155,19 @@ ftrace_call\suffix: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_call\suffix ftrace_graph_call\suffix: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif mcount_exit .endm .macro __ftrace_graph_caller +#ifdef CONFIG_UNWINDER_FRAME_POINTER sub r0, fp, #4 @ &lr of instrumented routine (&parent) +#else + add r0, sp, #20 +#endif #ifdef CONFIG_DYNAMIC_FTRACE @ called from __ftrace_caller, saved in mcount_enter ldr r1, [sp, #16] @ instrumented routine (func) @@ -178,7 +176,8 @@ ftrace_graph_call\suffix: @ called from __mcount, untouched in lr mcount_adjust_addr r1, lr @ instrumented routine (func) #endif - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #24 bl prepare_ftrace_return mcount_exit .endm @@ -202,16 +201,17 @@ ftrace_graph_call\suffix: .endm .macro mcount_exit - ldmia sp!, {r0-r3, ip, lr} - ret ip + ldmia sp!, {r0-r3} + ldr lr, [sp, #4] + ldr pc, [sp], #8 .endm ENTRY(__gnu_mcount_nc) UNWIND(.fnstart) #ifdef CONFIG_DYNAMIC_FTRACE - mov ip, lr - ldmia sp!, {lr} - ret ip + push {lr} + ldr lr, [sp, #4] + ldr pc, [sp], #8 #else __mcount #endif @@ -256,17 +256,33 @@ ENDPROC(ftrace_graph_regs_caller) .purgem mcount_exit #ifdef CONFIG_FUNCTION_GRAPH_TRACER - .globl return_to_handler -return_to_handler: +ENTRY(return_to_handler) stmdb sp!, {r0-r3} - mov r0, fp @ frame pointer + add r0, sp, #16 @ sp at exit of instrumented routine bl ftrace_return_to_handler mov lr, r0 @ r0 has real ret addr ldmia sp!, {r0-r3} ret lr +ENDPROC(return_to_handler) #endif ENTRY(ftrace_stub) .Lftrace_stub: ret lr ENDPROC(ftrace_stub) + +#ifdef CONFIG_DYNAMIC_FTRACE + + __INIT + + .macro init_tramp, dst:req +ENTRY(\dst\()_from_init) + ldr pc, =\dst +ENDPROC(\dst\()_from_init) + .endm + + init_tramp ftrace_caller +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + init_tramp ftrace_regs_caller +#endif +#endif diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index ae24dd54e9ef..9a1dc142f782 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -292,12 +292,18 @@ .macro restore_user_regs, fast = 0, offset = 0 -#if defined(CONFIG_CPU_32v6K) && !defined(CONFIG_CPU_V6) +#if defined(CONFIG_CPU_32v6K) && \ + (!defined(CONFIG_CPU_V6) || defined(CONFIG_SMP)) +#ifdef CONFIG_CPU_V6 +ALT_SMP(nop) +ALT_UP_B(.L1_\@) +#endif @ The TLS register update is deferred until return to user space so we @ can use it for other things while running in the kernel - get_thread_info r1 + mrc p15, 0, r1, c13, c0, 3 @ get current_thread_info pointer ldr r1, [r1, #TI_TP_VALUE] mcr p15, 0, r1, c13, c0, 3 @ set TLS register +.L1_\@: #endif uaccess_enable r1, isb=0 @@ -423,3 +429,40 @@ scno .req r7 @ syscall number tbl .req r8 @ syscall table pointer why .req r8 @ Linux syscall (!= 0) tsk .req r9 @ current thread_info + + .macro do_overflow_check, frame_size:req +#ifdef CONFIG_VMAP_STACK + @ + @ Test whether the SP has overflowed. Task and IRQ stacks are aligned + @ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be + @ zero. + @ +ARM( tst sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( tst r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( it ne ) + bne .Lstack_overflow_check\@ + + .pushsection .text +.Lstack_overflow_check\@: + @ + @ The stack pointer is not pointing to a valid vmap'ed stack, but it + @ may be pointing into the linear map instead, which may happen if we + @ are already running from the overflow stack. We cannot detect overflow + @ in such cases so just carry on. + @ + str ip, [r0, #12] @ Stash IP on the mode stack + ldr_va ip, high_memory @ Start of VMALLOC space +ARM( cmp sp, ip ) @ SP in vmalloc space? +THUMB( cmp r1, ip ) +THUMB( itt lo ) + ldrlo ip, [r0, #12] @ Restore IP + blo .Lout\@ @ Carry on + +THUMB( sub r1, sp, r1 ) @ Restore original R1 +THUMB( sub sp, r1 ) @ Restore original SP + add sp, sp, #\frame_size @ Undo svc_entry's SP change + b __bad_stack @ Handle VMAP stack overflow + .popsection +.Lout\@: +#endif + .endm diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 7bde93c10962..de8a60363c85 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -39,16 +39,25 @@ __irq_entry: @ @ Invoke the IRQ handler @ - mrs r0, ipsr - ldr r1, =V7M_xPSR_EXCEPTIONNO - and r0, r1 - sub r0, #16 - mov r1, sp - stmdb sp!, {lr} - @ routine called with r0 = irq number, r1 = struct pt_regs * - bl nvic_handle_irq - - pop {lr} + mov r0, sp + ldr_this_cpu sp, irq_stack_ptr, r1, r2 + + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r2, sp, r0 @ SP above bottom of IRQ stack? + rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + movcs sp, r0 + + push {r0, lr} @ preserve LR and original SP + + @ routine called with r0 = struct pt_regs * + bl generic_handle_arch_irq + + pop {r0, lr} + mov sp, r0 + @ @ Check for any pending work if returning to user @ @@ -101,15 +110,17 @@ ENTRY(__switch_to) str sp, [ip], #4 str lr, [ip], #4 mov r5, r0 + mov r6, r2 @ Preserve 'next' add r4, r2, #TI_CPU_SAVE ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain - mov ip, r4 mov r0, r5 - ldmia ip!, {r4 - r11} @ Load all regs saved previously - ldr sp, [ip] - ldr pc, [ip, #4]! + mov r1, r6 + ldmia r4, {r4 - r12, lr} @ Load all regs saved previously + set_current r1, r2 + mov sp, ip + bx lr .fnend ENDPROC(__switch_to) diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a006585e1c09..83cc068586bc 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -22,12 +22,24 @@ #include <asm/ftrace.h> #include <asm/insn.h> #include <asm/set_memory.h> +#include <asm/stacktrace.h> #include <asm/patch.h> +/* + * The compiler emitted profiling hook consists of + * + * PUSH {LR} + * BL __gnu_mcount_nc + * + * To turn this combined sequence into a NOP, we need to restore the value of + * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not + * modified anyway, and reloading LR from memory is highly likely to be less + * efficient. + */ #ifdef CONFIG_THUMB2_KERNEL -#define NOP 0xf85deb04 /* pop.w {lr} */ +#define NOP 0xf10d0d04 /* add.w sp, sp, #4 */ #else -#define NOP 0xe8bd4000 /* pop {lr} */ +#define NOP 0xe28dd004 /* add sp, sp, #4 */ #endif #ifdef CONFIG_DYNAMIC_FTRACE @@ -51,9 +63,20 @@ static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) return NOP; } -static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +void ftrace_caller_from_init(void); +void ftrace_regs_caller_from_init(void); + +static unsigned long __ref adjust_address(struct dyn_ftrace *rec, + unsigned long addr) { - return addr; + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) || + system_state >= SYSTEM_FREEING_INITMEM || + likely(!is_kernel_inittext(rec->ip))) + return addr; + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || + addr == (unsigned long)&ftrace_caller) + return (unsigned long)&ftrace_caller_from_init; + return (unsigned long)&ftrace_regs_caller_from_init; } int ftrace_arch_code_modify_prepare(void) @@ -189,15 +212,23 @@ int ftrace_make_nop(struct module *mod, #endif new = ftrace_nop_replace(rec); - ret = ftrace_modify_code(ip, old, new, true); + /* + * Locations in .init.text may call __gnu_mcount_mc via a linker + * emitted veneer if they are too far away from its implementation, and + * so validation may fail spuriously in such cases. Let's work around + * this by omitting those from validation. + */ + ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip)); return ret; } #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER +asmlinkage void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, - unsigned long frame_pointer) + unsigned long frame_pointer, + unsigned long stack_pointer) { unsigned long return_hooker = (unsigned long) &return_to_handler; unsigned long old; @@ -205,6 +236,23 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { + /* FP points one word below parent's top of stack */ + frame_pointer += 4; + } else { + struct stackframe frame = { + .fp = frame_pointer, + .sp = stack_pointer, + .lr = self_addr, + .pc = self_addr, + }; + if (unwind_frame(&frame) < 0) + return; + if (frame.lr != self_addr) + parent = frame.lr_addr; + frame_pointer = frame.sp; + } + old = *parent; *parent = return_hooker; @@ -225,7 +273,7 @@ static int __ftrace_modify_caller(unsigned long *callsite, unsigned long caller_fn = (unsigned long) func; unsigned long pc = (unsigned long) callsite; unsigned long branch = arm_gen_branch(pc, caller_fn); - unsigned long nop = 0xe1a00000; /* mov r0, r0 */ + unsigned long nop = arm_gen_nop(); unsigned long old = enable ? nop : branch; unsigned long new = enable ? branch : nop; diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index da18e0a17dc2..42cae73fcc19 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -105,10 +105,8 @@ __mmap_switched: mov r1, #0 bl __memset @ clear .bss -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO adr_l r0, init_task @ get swapper task_struct - set_current r0 -#endif + set_current r0, r1 ldmia r4, {r0, r1, r2, r3} str r9, [r0] @ Save processor ID diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index c04dd94630c7..500612d3da2e 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -424,6 +424,13 @@ ENDPROC(secondary_startup) ENDPROC(secondary_startup_arm) ENTRY(__secondary_switched) +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif adr_l r7, secondary_data + 12 @ get secondary_data.stack ldr sp, [r7] ldr r0, [r7, #4] @ get secondary_data.task diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index b79975bd988c..5c6f8d11a3ce 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -36,13 +36,53 @@ #include <asm/hardware/cache-l2x0.h> #include <asm/hardware/cache-uniphier.h> #include <asm/outercache.h> +#include <asm/softirq_stack.h> #include <asm/exception.h> #include <asm/mach/arch.h> #include <asm/mach/irq.h> #include <asm/mach/time.h> +#include "reboot.h" + unsigned long irq_err_count; +#ifdef CONFIG_IRQSTACKS + +asmlinkage DEFINE_PER_CPU_READ_MOSTLY(u8 *, irq_stack_ptr); + +static void __init init_irq_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + if (!IS_ENABLED(CONFIG_VMAP_STACK)) + stack = (u8 *)__get_free_pages(GFP_KERNEL, + THREAD_SIZE_ORDER); + else + stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, + THREADINFO_GFP, NUMA_NO_NODE, + __builtin_return_address(0)); + + if (WARN_ON(!stack)) + break; + per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE]; + } +} + +static void ____do_softirq(void *arg) +{ + __do_softirq(); +} + +void do_softirq_own_stack(void) +{ + call_with_stack(____do_softirq, NULL, + __this_cpu_read(irq_stack_ptr)); +} + +#endif + int arch_show_interrupts(struct seq_file *p, int prec) { #ifdef CONFIG_FIQ @@ -80,27 +120,14 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs) ack_bad_irq(irq); } -/* - * asm_do_IRQ is the interface to be used from assembly code. - */ -asmlinkage void __exception_irq_entry -asm_do_IRQ(unsigned int irq, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - - irq_enter(); - old_regs = set_irq_regs(regs); - - handle_IRQ(irq, regs); - - set_irq_regs(old_regs); - irq_exit(); -} - void __init init_IRQ(void) { int ret; +#ifdef CONFIG_IRQSTACKS + init_irq_stacks(); +#endif + if (IS_ENABLED(CONFIG_OF) && !machine_desc->init_irq) irqchip_init(); else diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c index 7bd30c0a4280..22f937e6f3ff 100644 --- a/arch/arm/kernel/kgdb.c +++ b/arch/arm/kernel/kgdb.c @@ -154,22 +154,38 @@ static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int instr) return 0; } -static struct undef_hook kgdb_brkpt_hook = { +static struct undef_hook kgdb_brkpt_arm_hook = { .instr_mask = 0xffffffff, .instr_val = KGDB_BREAKINST, - .cpsr_mask = MODE_MASK, + .cpsr_mask = PSR_T_BIT | MODE_MASK, .cpsr_val = SVC_MODE, .fn = kgdb_brk_fn }; -static struct undef_hook kgdb_compiled_brkpt_hook = { +static struct undef_hook kgdb_brkpt_thumb_hook = { + .instr_mask = 0xffff, + .instr_val = KGDB_BREAKINST & 0xffff, + .cpsr_mask = PSR_T_BIT | MODE_MASK, + .cpsr_val = PSR_T_BIT | SVC_MODE, + .fn = kgdb_brk_fn +}; + +static struct undef_hook kgdb_compiled_brkpt_arm_hook = { .instr_mask = 0xffffffff, .instr_val = KGDB_COMPILED_BREAK, - .cpsr_mask = MODE_MASK, + .cpsr_mask = PSR_T_BIT | MODE_MASK, .cpsr_val = SVC_MODE, .fn = kgdb_compiled_brk_fn }; +static struct undef_hook kgdb_compiled_brkpt_thumb_hook = { + .instr_mask = 0xffff, + .instr_val = KGDB_COMPILED_BREAK & 0xffff, + .cpsr_mask = PSR_T_BIT | MODE_MASK, + .cpsr_val = PSR_T_BIT | SVC_MODE, + .fn = kgdb_compiled_brk_fn +}; + static int __kgdb_notify(struct die_args *args, unsigned long cmd) { struct pt_regs *regs = args->regs; @@ -210,8 +226,10 @@ int kgdb_arch_init(void) if (ret != 0) return ret; - register_undef_hook(&kgdb_brkpt_hook); - register_undef_hook(&kgdb_compiled_brkpt_hook); + register_undef_hook(&kgdb_brkpt_arm_hook); + register_undef_hook(&kgdb_brkpt_thumb_hook); + register_undef_hook(&kgdb_compiled_brkpt_arm_hook); + register_undef_hook(&kgdb_compiled_brkpt_thumb_hook); return 0; } @@ -224,8 +242,10 @@ int kgdb_arch_init(void) */ void kgdb_arch_exit(void) { - unregister_undef_hook(&kgdb_brkpt_hook); - unregister_undef_hook(&kgdb_compiled_brkpt_hook); + unregister_undef_hook(&kgdb_brkpt_arm_hook); + unregister_undef_hook(&kgdb_brkpt_thumb_hook); + unregister_undef_hook(&kgdb_compiled_brkpt_arm_hook); + unregister_undef_hook(&kgdb_compiled_brkpt_thumb_hook); unregister_die_notifier(&kgdb_notifier); } diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index beac45e89ba6..549abcedf795 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -68,6 +68,44 @@ bool module_exit_section(const char *name) strstarts(name, ".ARM.exidx.exit"); } +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS +/* + * This implements the partitioning algorithm for group relocations as + * documented in the ARM AArch32 ELF psABI (IHI 0044). + * + * A single PC-relative symbol reference is divided in up to 3 add or subtract + * operations, where the final one could be incorporated into a load/store + * instruction with immediate offset. E.g., + * + * ADD Rd, PC, #... or ADD Rd, PC, #... + * ADD Rd, Rd, #... ADD Rd, Rd, #... + * LDR Rd, [Rd, #...] ADD Rd, Rd, #... + * + * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is + * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of + * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for + * any in-kernel symbol reference (unless module PLTs are being used). + * + * The main advantage of this approach over the typical pattern using a literal + * load is that literal loads may miss in the D-cache, and generally lead to + * lower cache efficiency for variables that are referenced often from many + * different places in the code. + */ +static u32 get_group_rem(u32 group, u32 *offset) +{ + u32 val = *offset; + u32 shift; + do { + shift = val ? (31 - __fls(val)) & ~1 : 32; + *offset = val; + if (!val) + break; + val &= 0xffffff >> shift; + } while (group--); + return shift; +} +#endif + int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relindex, struct module *module) @@ -82,6 +120,9 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned long loc; Elf32_Sym *sym; const char *symname; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS + u32 shift, group = 1; +#endif s32 offset; u32 tmp; #ifdef CONFIG_THUMB2_KERNEL @@ -212,6 +253,55 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, *(u32 *)loc = __opcode_to_mem_arm(tmp); break; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS + case R_ARM_ALU_PC_G0_NC: + group = 0; + fallthrough; + case R_ARM_ALU_PC_G1_NC: + tmp = __mem_to_opcode_arm(*(u32 *)loc); + offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7); + if (tmp & BIT(22)) + offset = -offset; + offset += sym->st_value - loc; + if (offset < 0) { + offset = -offset; + tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode + } else { + tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode + } + + shift = get_group_rem(group, &offset); + if (shift < 24) { + offset >>= 24 - shift; + offset |= (shift + 8) << 7; + } + *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); + break; + + case R_ARM_LDR_PC_G2: + tmp = __mem_to_opcode_arm(*(u32 *)loc); + offset = tmp & 0xfff; + if (~tmp & BIT(23)) // U bit cleared? + offset = -offset; + offset += sym->st_value - loc; + if (offset < 0) { + offset = -offset; + tmp &= ~BIT(23); // clear U bit + } else { + tmp |= BIT(23); // set U bit + } + get_group_rem(2, &offset); + + if (offset > 0xfff) { + pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", + module->name, relindex, i, symname, + ELF32_R_TYPE(rel->r_info), loc, + sym->st_value); + return -ENOEXEC; + } + *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); + break; +#endif #ifdef CONFIG_THUMB2_KERNEL case R_ARM_THM_CALL: case R_ARM_THM_JUMP24: diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index d47159f3791c..0617af11377f 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -36,7 +36,7 @@ #include "signal.h" -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) DEFINE_PER_CPU(struct task_struct *, __entry_task); #endif @@ -46,6 +46,11 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif +#ifndef CONFIG_CURRENT_POINTER_IN_TPIDRURO +asmlinkage struct task_struct *__current; +EXPORT_SYMBOL(__current); +#endif + static const char *processor_modes[] __maybe_unused = { "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c index 00c11579406c..8aac1e10b117 100644 --- a/arch/arm/kernel/return_address.c +++ b/arch/arm/kernel/return_address.c @@ -41,7 +41,8 @@ void *return_address(unsigned int level) frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)return_address; +here: + frame.pc = (unsigned long)&&here; #ifdef CONFIG_KRETPROBES frame.kr_cur = NULL; frame.tsk = current; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 284a80c0b6e1..039feb7cd590 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -141,10 +141,10 @@ EXPORT_SYMBOL(outer_cache); int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN; struct stack { - u32 irq[3]; - u32 abt[3]; - u32 und[3]; - u32 fiq[3]; + u32 irq[4]; + u32 abt[4]; + u32 und[4]; + u32 fiq[4]; } ____cacheline_aligned; #ifndef CONFIG_CPU_V7M diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index 43077e11dafd..a86a1d4f3461 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -67,6 +67,12 @@ ENTRY(__cpu_suspend) ldr r4, =cpu_suspend_size #endif mov r5, sp @ current virtual SP +#ifdef CONFIG_VMAP_STACK + @ Run the suspend code from the overflow stack so we don't have to rely + @ on vmalloc-to-phys conversions anywhere in the arch suspend code. + @ The original SP value captured in R5 will be restored on the way out. + ldr_this_cpu sp, overflow_stack_ptr, r6, r7 +#endif add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn sub sp, sp, r4 @ allocate CPU state on stack ldr r3, =sleep_save_sp @@ -113,6 +119,13 @@ ENTRY(cpu_resume_mmu) ENDPROC(cpu_resume_mmu) .popsection cpu_resume_after_mmu: +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif bl cpu_init @ restore the und/abt/irq banked regs mov r0, #0 @ return zero on success ldmfd sp!, {r4 - r11, pc} diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 97ee6b1567e9..73fc645fc4c7 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -400,6 +400,12 @@ static void smp_store_cpu_info(unsigned int cpuid) check_cpu_icache_size(cpuid); } +static void set_current(struct task_struct *cur) +{ + /* Set TPIDRURO */ + asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); +} + /* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. @@ -628,11 +634,6 @@ static void ipi_complete(unsigned int cpu) /* * Main handler for inter-processor interrupts */ -asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) -{ - handle_IPI(ipinr, regs); -} - static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c new file mode 100644 index 000000000000..0dcefc36fb7a --- /dev/null +++ b/arch/arm/kernel/spectre.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/bpf.h> +#include <linux/cpu.h> +#include <linux/device.h> + +#include <asm/spectre.h> + +static bool _unprivileged_ebpf_enabled(void) +{ +#ifdef CONFIG_BPF_SYSCALL + return !sysctl_unprivileged_bpf_disabled; +#else + return false; +#endif +} + +ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "Mitigation: __user pointer sanitization\n"); +} + +static unsigned int spectre_v2_state; +static unsigned int spectre_v2_methods; + +void spectre_v2_update_state(unsigned int state, unsigned int method) +{ + if (state > spectre_v2_state) + spectre_v2_state = state; + spectre_v2_methods |= method; +} + +ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *method; + + if (spectre_v2_state == SPECTRE_UNAFFECTED) + return sprintf(buf, "%s\n", "Not affected"); + + if (spectre_v2_state != SPECTRE_MITIGATED) + return sprintf(buf, "%s\n", "Vulnerable"); + + if (_unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + + switch (spectre_v2_methods) { + case SPECTRE_V2_METHOD_BPIALL: + method = "Branch predictor hardening"; + break; + + case SPECTRE_V2_METHOD_ICIALLU: + method = "I-cache invalidation"; + break; + + case SPECTRE_V2_METHOD_SMC: + case SPECTRE_V2_METHOD_HVC: + method = "Firmware call"; + break; + + case SPECTRE_V2_METHOD_LOOP8: + method = "History overwrite"; + break; + + default: + method = "Multiple mitigations"; + break; + } + + return sprintf(buf, "Mitigation: %s\n", method); +} diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index 75e905508f27..b5efecb3d730 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -160,7 +160,8 @@ static noinline void __save_stack_trace(struct task_struct *tsk, frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)__save_stack_trace; +here: + frame.pc = (unsigned long)&&here; } #ifdef CONFIG_KRETPROBES frame.kr_cur = NULL; diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index 6166ba38bf99..b74bfcf94fb1 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -195,7 +195,7 @@ static int swp_handler(struct pt_regs *regs, unsigned int instr) destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data); /* Check access in reasonable access range for both SWP and SWPB */ - if (!access_ok((address & ~3), 4)) { + if (!access_ok((void __user *)(address & ~3), 4)) { pr_debug("SWP{B} emulation: access to %p not allowed!\n", (void *)address); res = -EFAULT; diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index da04ed85855a..9283dc65be31 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -30,11 +30,13 @@ #include <linux/atomic.h> #include <asm/cacheflush.h> #include <asm/exception.h> +#include <asm/spectre.h> #include <asm/unistd.h> #include <asm/traps.h> #include <asm/ptrace.h> #include <asm/unwind.h> #include <asm/tls.h> +#include <asm/stacktrace.h> #include <asm/system_misc.h> #include <asm/opcodes.h> @@ -60,13 +62,24 @@ static int __init user_debug_setup(char *str) __setup("user_debug=", user_debug_setup); #endif -static void dump_mem(const char *, const char *, unsigned long, unsigned long); - void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame, const char *loglvl) { unsigned long end = frame + 4 + sizeof(struct pt_regs); + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) && + IS_ENABLED(CONFIG_CC_IS_GCC) && + end > ALIGN(frame, THREAD_SIZE)) { + /* + * If we are walking past the end of the stack, it may be due + * to the fact that we are on an IRQ or overflow stack. In this + * case, we can load the address of the other stack from the + * frame record. + */ + frame = ((unsigned long *)frame)[-2] - 4; + end = frame + 4 + sizeof(struct pt_regs); + } + #ifndef CONFIG_KALLSYMS printk("%sFunction entered at [<%08lx>] from [<%08lx>]\n", loglvl, where, from); @@ -110,7 +123,8 @@ void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl) static int verify_stack(unsigned long sp) { if (sp < PAGE_OFFSET || - (sp > (unsigned long)high_memory && high_memory != NULL)) + (!IS_ENABLED(CONFIG_VMAP_STACK) && + sp > (unsigned long)high_memory && high_memory != NULL)) return -EFAULT; return 0; @@ -120,8 +134,8 @@ static int verify_stack(unsigned long sp) /* * Dump out the contents of some memory nicely... */ -static void dump_mem(const char *lvl, const char *str, unsigned long bottom, - unsigned long top) +void dump_mem(const char *lvl, const char *str, unsigned long bottom, + unsigned long top) { unsigned long first; int i; @@ -280,7 +294,8 @@ static int __die(const char *str, int err, struct pt_regs *regs) if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk)); + ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE); dump_backtrace(regs, tsk, KERN_EMERG); dump_instr(KERN_EMERG, regs); } @@ -576,7 +591,7 @@ do_cache_op(unsigned long start, unsigned long end, int flags) if (end < start || flags) return -EINVAL; - if (!access_ok(start, end - start)) + if (!access_ok((void __user *)start, end - start)) return -EFAULT; return __do_cache_op(start, end); @@ -789,10 +804,59 @@ static inline void __init kuser_init(void *vectors) } #endif +#ifndef CONFIG_CPU_V7M +static void copy_from_lma(void *vma, void *lma_start, void *lma_end) +{ + memcpy(vma, lma_start, lma_end - lma_start); +} + +static void flush_vectors(void *vma, size_t offset, size_t size) +{ + unsigned long start = (unsigned long)vma + offset; + unsigned long end = start + size; + + flush_icache_range(start, end); +} + +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +int spectre_bhb_update_vectors(unsigned int method) +{ + extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; + extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; + void *vec_start, *vec_end; + + if (system_state >= SYSTEM_FREEING_INITMEM) { + pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", + smp_processor_id()); + return SPECTRE_VULNERABLE; + } + + switch (method) { + case SPECTRE_V2_METHOD_LOOP8: + vec_start = __vectors_bhb_loop8_start; + vec_end = __vectors_bhb_loop8_end; + break; + + case SPECTRE_V2_METHOD_BPIALL: + vec_start = __vectors_bhb_bpiall_start; + vec_end = __vectors_bhb_bpiall_end; + break; + + default: + pr_err("CPU%u: unknown Spectre BHB state %d\n", + smp_processor_id(), method); + return SPECTRE_VULNERABLE; + } + + copy_from_lma(vectors_page, vec_start, vec_end); + flush_vectors(vectors_page, 0, vec_end - vec_start); + + return SPECTRE_MITIGATED; +} +#endif + void __init early_trap_init(void *vectors_base) { -#ifndef CONFIG_CPU_V7M - unsigned long vectors = (unsigned long)vectors_base; extern char __stubs_start[], __stubs_end[]; extern char __vectors_start[], __vectors_end[]; unsigned i; @@ -813,17 +877,87 @@ void __init early_trap_init(void *vectors_base) * into the vector page, mapped at 0xffff0000, and ensure these * are visible to the instruction stream. */ - memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); - memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start); + copy_from_lma(vectors_base, __vectors_start, __vectors_end); + copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end); kuser_init(vectors_base); - flush_icache_range(vectors, vectors + PAGE_SIZE * 2); + flush_vectors(vectors_base, 0, PAGE_SIZE * 2); +} #else /* ifndef CONFIG_CPU_V7M */ +void __init early_trap_init(void *vectors_base) +{ /* * on V7-M there is no need to copy the vector table to a dedicated * memory area. The address is configurable and so a table in the kernel * image can be used. */ +} +#endif + +#ifdef CONFIG_VMAP_STACK + +DECLARE_PER_CPU(u8 *, irq_stack_ptr); + +asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr); + +static int __init allocate_overflow_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + stack = (u8 *)__get_free_page(GFP_KERNEL); + if (WARN_ON(!stack)) + return -ENOMEM; + per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE]; + } + return 0; +} +early_initcall(allocate_overflow_stacks); + +asmlinkage void handle_bad_stack(struct pt_regs *regs) +{ + unsigned long tsk_stk = (unsigned long)current->stack; +#ifdef CONFIG_IRQSTACKS + unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); #endif + unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); + + console_verbose(); + pr_emerg("Insufficient stack space to handle exception!"); + + pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", + tsk_stk, tsk_stk + THREAD_SIZE); +#ifdef CONFIG_IRQSTACKS + pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", + irq_stk - THREAD_SIZE, irq_stk); +#endif + pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", + ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); + + die("kernel stack overflow", regs, 0); +} + +#ifndef CONFIG_ARM_LPAE +/* + * Normally, we rely on the logic in do_translation_fault() to update stale PMD + * entries covering the vmalloc space in a task's page tables when it first + * accesses the region in question. Unfortunately, this is not sufficient when + * the task stack resides in the vmalloc region, as do_translation_fault() is a + * C function that needs a stack to run. + * + * So we need to ensure that these PMD entries are up to date *before* the MM + * switch. As we already have some logic in the MM switch path that takes care + * of this, let's trigger it by bumping the counter every time the core vmalloc + * code modifies a PMD entry in the vmalloc region. Use release semantics on + * the store so that other CPUs observing the counter's new value are + * guaranteed to see the updated page table entries as well. + */ +void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +{ + if (start < VMALLOC_END && end > VMALLOC_START) + atomic_inc_return_release(&init_mm.context.vmalloc_seq); } +#endif +#endif diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 59fdf257bf8b..a37ea6c772cd 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -33,6 +33,8 @@ #include <asm/traps.h> #include <asm/unwind.h> +#include "reboot.h" + /* Dummy functions to avoid linker complaints */ void __aeabi_unwind_cpp_pr0(void) { @@ -53,6 +55,7 @@ struct unwind_ctrl_block { unsigned long vrs[16]; /* virtual register set */ const unsigned long *insn; /* pointer to the current instructions word */ unsigned long sp_high; /* highest value of sp allowed */ + unsigned long *lr_addr; /* address of LR value on the stack */ /* * 1 : check for stack overflow for each register pop. * 0 : save overhead if there is plenty of stack remaining. @@ -237,6 +240,8 @@ static int unwind_pop_register(struct unwind_ctrl_block *ctrl, * from being tracked by KASAN. */ ctrl->vrs[reg] = READ_ONCE_NOCHECK(*(*vsp)); + if (reg == 14) + ctrl->lr_addr = *vsp; (*vsp)++; return URC_OK; } @@ -256,8 +261,9 @@ static int unwind_exec_pop_subset_r4_to_r13(struct unwind_ctrl_block *ctrl, mask >>= 1; reg++; } - if (!load_sp) + if (!load_sp) { ctrl->vrs[SP] = (unsigned long)vsp; + } return URC_OK; } @@ -313,9 +319,9 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if ((insn & 0xc0) == 0x00) ctrl->vrs[SP] += ((insn & 0x3f) << 2) + 4; - else if ((insn & 0xc0) == 0x40) + else if ((insn & 0xc0) == 0x40) { ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; - else if ((insn & 0xf0) == 0x80) { + } else if ((insn & 0xf0) == 0x80) { unsigned long mask; insn = (insn << 8) | unwind_get_byte(ctrl); @@ -330,9 +336,9 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if (ret) goto error; } else if ((insn & 0xf0) == 0x90 && - (insn & 0x0d) != 0x0d) + (insn & 0x0d) != 0x0d) { ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; - else if ((insn & 0xf0) == 0xa0) { + } else if ((insn & 0xf0) == 0xa0) { ret = unwind_exec_pop_r4_to_rN(ctrl, insn); if (ret) goto error; @@ -375,23 +381,22 @@ error: */ int unwind_frame(struct stackframe *frame) { - unsigned long low; const struct unwind_idx *idx; struct unwind_ctrl_block ctrl; + unsigned long sp_low; /* store the highest address on the stack to avoid crossing it*/ - low = frame->sp; - ctrl.sp_high = ALIGN(low, THREAD_SIZE); + sp_low = frame->sp; + ctrl.sp_high = ALIGN(sp_low - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE; pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); - if (!kernel_text_address(frame->pc)) - return -URC_FAILURE; - idx = unwind_find_idx(frame->pc); if (!idx) { - pr_warn("unwind: Index not found %08lx\n", frame->pc); + if (frame->pc && kernel_text_address(frame->pc)) + pr_warn("unwind: Index not found %08lx\n", frame->pc); return -URC_FAILURE; } @@ -403,7 +408,20 @@ int unwind_frame(struct stackframe *frame) if (idx->insn == 1) /* can't unwind */ return -URC_FAILURE; - else if ((idx->insn & 0x80000000) == 0) + else if (frame->pc == prel31_to_addr(&idx->addr_offset)) { + /* + * Unwinding is tricky when we're halfway through the prologue, + * since the stack frame that the unwinder expects may not be + * fully set up yet. However, one thing we do know for sure is + * that if we are unwinding from the very first instruction of + * a function, we are still effectively in the stack frame of + * the caller, and the unwind info has no relevance yet. + */ + if (frame->pc == frame->lr) + return -URC_FAILURE; + frame->pc = frame->lr; + return URC_OK; + } else if ((idx->insn & 0x80000000) == 0) /* prel31 to the unwind table */ ctrl.insn = (unsigned long *)prel31_to_addr(&idx->insn); else if ((idx->insn & 0xff000000) == 0x80000000) @@ -430,6 +448,16 @@ int unwind_frame(struct stackframe *frame) ctrl.check_each_pop = 0; + if (prel31_to_addr(&idx->addr_offset) == (u32)&call_with_stack) { + /* + * call_with_stack() is the only place where we permit SP to + * jump from one stack to another, and since we know it is + * guaranteed to happen, set up the SP bounds accordingly. + */ + sp_low = frame->fp; + ctrl.sp_high = ALIGN(frame->fp, THREAD_SIZE); + } + while (ctrl.entries > 0) { int urc; if ((ctrl.sp_high - ctrl.vrs[SP]) < sizeof(ctrl.vrs)) @@ -437,7 +465,7 @@ int unwind_frame(struct stackframe *frame) urc = unwind_exec_insn(&ctrl); if (urc < 0) return urc; - if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= ctrl.sp_high) + if (ctrl.vrs[SP] < sp_low || ctrl.vrs[SP] > ctrl.sp_high) return -URC_FAILURE; } @@ -452,6 +480,7 @@ int unwind_frame(struct stackframe *frame) frame->sp = ctrl.vrs[SP]; frame->lr = ctrl.vrs[LR]; frame->pc = ctrl.vrs[PC]; + frame->lr_addr = ctrl.lr_addr; return URC_OK; } @@ -475,7 +504,12 @@ void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)unwind_backtrace; + /* We are saving the stack and execution state at this + * point, so we should ensure that frame.pc is within + * this block of code. + */ +here: + frame.pc = (unsigned long)&&here; } else { /* task blocked in __switch_to */ frame.fp = thread_saved_fp(tsk); diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index f02d617e3359..aa12b65a7fd6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -138,12 +138,12 @@ SECTIONS #ifdef CONFIG_STRICT_KERNEL_RWX . = ALIGN(1<<SECTION_SHIFT); #else - . = ALIGN(THREAD_SIZE); + . = ALIGN(THREAD_ALIGN); #endif __init_end = .; _sdata = .; - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) + RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN) _edata = .; BSS_SECTION(0, 0, 0) diff --git a/arch/arm/lib/backtrace-clang.S b/arch/arm/lib/backtrace-clang.S index 5b2cdb1003e3..290c52a60fc6 100644 --- a/arch/arm/lib/backtrace-clang.S +++ b/arch/arm/lib/backtrace-clang.S @@ -144,7 +144,7 @@ for_each_frame: tst frame, mask @ Check for address exceptions */ 1003: ldr sv_lr, [sv_fp, #4] @ get saved lr from next frame - ldr r0, [sv_lr, #-4] @ get call instruction +1004: ldr r0, [sv_lr, #-4] @ get call instruction ldr r3, .Lopcode+4 and r2, r3, r0 @ is this a bl call teq r2, r3 @@ -164,7 +164,7 @@ finished_setup: /* * Print the function (sv_pc) and where it was called from (sv_lr). */ -1004: mov r0, sv_pc + mov r0, sv_pc mov r1, sv_lr mov r2, frame @@ -197,6 +197,14 @@ finished_setup: cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad @@ -210,7 +218,7 @@ ENDPROC(c_backtrace) .long 1001b, 1006b .long 1002b, 1006b .long 1003b, 1006b - .long 1004b, 1006b + .long 1004b, finished_setup .long 1005b, 1006b .popsection diff --git a/arch/arm/lib/backtrace.S b/arch/arm/lib/backtrace.S index e8408f22d4dc..293a2716bd20 100644 --- a/arch/arm/lib/backtrace.S +++ b/arch/arm/lib/backtrace.S @@ -98,6 +98,14 @@ for_each_frame: tst frame, mask @ Check for address exceptions cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad diff --git a/arch/arm/lib/call_with_stack.S b/arch/arm/lib/call_with_stack.S index 28b0341ae786..0a268a6c513c 100644 --- a/arch/arm/lib/call_with_stack.S +++ b/arch/arm/lib/call_with_stack.S @@ -8,25 +8,42 @@ #include <linux/linkage.h> #include <asm/assembler.h> +#include <asm/unwind.h> /* * void call_with_stack(void (*fn)(void *), void *arg, void *sp) * * Change the stack to that pointed at by sp, then invoke fn(arg) with * the new stack. + * + * The sequence below follows the APCS frame convention for frame pointer + * unwinding, and implements the unwinder annotations needed by the EABI + * unwinder. */ -ENTRY(call_with_stack) - str sp, [r2, #-4]! - str lr, [r2, #-4]! +ENTRY(call_with_stack) +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + mov ip, sp + push {fp, ip, lr, pc} + sub fp, ip, #4 +#else +UNWIND( .fnstart ) +UNWIND( .save {fpreg, lr} ) + push {fpreg, lr} +UNWIND( .setfp fpreg, sp ) + mov fpreg, sp +#endif mov sp, r2 mov r2, r0 mov r0, r1 - badr lr, 1f - ret r2 + bl_r r2 -1: ldr lr, [sp] - ldr sp, [sp, #4] - ret lr +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + ldmdb fp, {fp, sp, pc} +#else + mov sp, fpreg + pop {fpreg, pc} +UNWIND( .fnend ) +#endif ENDPROC(call_with_stack) diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 480a20766137..270de7debd0f 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -91,18 +91,15 @@ strb\cond \reg, [\ptr], #1 .endm - .macro enter reg1 reg2 + .macro enter regs:vararg mov r3, #0 - stmdb sp!, {r0, r2, r3, \reg1, \reg2} +UNWIND( .save {r0, r2, r3, \regs} ) + stmdb sp!, {r0, r2, r3, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, r2, r3, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 + .macro exit regs:vararg add sp, sp, #8 - ldmfd sp!, {r0, \reg1, \reg2} + ldmfd sp!, {r0, \regs} .endm .text diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 810a805d36dc..8fbafb074fe9 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -69,13 +69,10 @@ * than one 32bit instruction in Thumb-2) */ - - UNWIND( .fnstart ) - enter r4, lr - UNWIND( .fnend ) - UNWIND( .fnstart ) - usave r4, lr @ in first stmdb block + enter r4, UNWIND(fpreg,) lr + UNWIND( .setfp fpreg, sp ) + UNWIND( mov fpreg, sp ) subs r2, r2, #4 blt 8f @@ -86,12 +83,7 @@ bne 10f 1: subs r2, r2, #(28) - stmfd sp!, {r5 - r8} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - usave r4, lr - UNWIND( .save {r5 - r8} ) @ in second stmfd block + stmfd sp!, {r5, r6, r8, r9} blt 5f CALGN( ands ip, r0, #31 ) @@ -110,9 +102,9 @@ PLD( pld [r1, #92] ) 3: PLD( pld [r1, #124] ) -4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f +4: ldr8w r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f subs r2, r2, #32 - str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f + str8w r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f bge 3b PLD( cmn r2, #96 ) PLD( bge 4b ) @@ -132,8 +124,8 @@ ldr1w r1, r4, abort=20f ldr1w r1, r5, abort=20f ldr1w r1, r6, abort=20f - ldr1w r1, r7, abort=20f ldr1w r1, r8, abort=20f + ldr1w r1, r9, abort=20f ldr1w r1, lr, abort=20f #if LDR1W_SHIFT < STR1W_SHIFT @@ -150,17 +142,14 @@ str1w r0, r4, abort=20f str1w r0, r5, abort=20f str1w r0, r6, abort=20f - str1w r0, r7, abort=20f str1w r0, r8, abort=20f + str1w r0, r9, abort=20f str1w r0, lr, abort=20f CALGN( bcs 2b ) -7: ldmfd sp!, {r5 - r8} - UNWIND( .fnend ) @ end of second stmfd block +7: ldmfd sp!, {r5, r6, r8, r9} - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block 8: movs r2, r2, lsl #31 ldr1b r1, r3, ne, abort=21f ldr1b r1, r4, cs, abort=21f @@ -169,7 +158,7 @@ str1b r0, r4, cs, abort=21f str1b r0, ip, cs, abort=21f - exit r4, pc + exit r4, UNWIND(fpreg,) pc 9: rsb ip, ip, #4 cmp ip, #2 @@ -189,13 +178,10 @@ ldr1w r1, lr, abort=21f beq 17f bgt 18f - UNWIND( .fnend ) .macro forward_copy_shift pull push - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block subs r2, r2, #28 blt 14f @@ -205,12 +191,8 @@ CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) -11: stmfd sp!, {r5 - r9} - UNWIND( .fnend ) +11: stmfd sp!, {r5, r6, r8 - r10} - UNWIND( .fnstart ) - usave r4, lr - UNWIND( .save {r5 - r9} ) @ in new second stmfd block PLD( pld [r1, #0] ) PLD( subs r2, r2, #96 ) PLD( pld [r1, #28] ) @@ -219,35 +201,32 @@ PLD( pld [r1, #92] ) 12: PLD( pld [r1, #124] ) -13: ldr4w r1, r4, r5, r6, r7, abort=19f +13: ldr4w r1, r4, r5, r6, r8, abort=19f mov r3, lr, lspull #\pull subs r2, r2, #32 - ldr4w r1, r8, r9, ip, lr, abort=19f + ldr4w r1, r9, r10, ip, lr, abort=19f orr r3, r3, r4, lspush #\push mov r4, r4, lspull #\pull orr r4, r4, r5, lspush #\push mov r5, r5, lspull #\pull orr r5, r5, r6, lspush #\push mov r6, r6, lspull #\pull - orr r6, r6, r7, lspush #\push - mov r7, r7, lspull #\pull - orr r7, r7, r8, lspush #\push + orr r6, r6, r8, lspush #\push mov r8, r8, lspull #\pull orr r8, r8, r9, lspush #\push mov r9, r9, lspull #\pull - orr r9, r9, ip, lspush #\push + orr r9, r9, r10, lspush #\push + mov r10, r10, lspull #\pull + orr r10, r10, ip, lspush #\push mov ip, ip, lspull #\pull orr ip, ip, lr, lspush #\push - str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f + str8w r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f bge 12b PLD( cmn r2, #96 ) PLD( bge 13b ) - ldmfd sp!, {r5 - r9} - UNWIND( .fnend ) @ end of the second stmfd block + ldmfd sp!, {r5, r6, r8 - r10} - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block 14: ands ip, r2, #28 beq 16f @@ -262,7 +241,6 @@ 16: sub r1, r1, #(\push / 8) b 8b - UNWIND( .fnend ) .endm @@ -273,6 +251,7 @@ 18: forward_copy_shift pull=24 push=8 + UNWIND( .fnend ) /* * Abort preamble and completion macros. @@ -282,13 +261,13 @@ */ .macro copy_abort_preamble -19: ldmfd sp!, {r5 - r9} +19: ldmfd sp!, {r5, r6, r8 - r10} b 21f -20: ldmfd sp!, {r5 - r8} +20: ldmfd sp!, {r5, r6, r8, r9} 21: .endm .macro copy_abort_end - ldmfd sp!, {r4, pc} + ldmfd sp!, {r4, UNWIND(fpreg,) pc} .endm diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 842ea5ede485..fac49e57cc0b 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -90,18 +90,15 @@ strusr \reg, \ptr, 1, \cond, abort=\abort .endm - .macro enter reg1 reg2 + .macro enter regs:vararg mov r3, #0 - stmdb sp!, {r0, r2, r3, \reg1, \reg2} +UNWIND( .save {r0, r2, r3, \regs} ) + stmdb sp!, {r0, r2, r3, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, r2, r3, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 + .macro exit regs:vararg add sp, sp, #8 - ldmfd sp!, {r0, \reg1, \reg2} + ldmfd sp!, {r0, \regs} .endm .text diff --git a/arch/arm/lib/memcpy.S b/arch/arm/lib/memcpy.S index e4caf48c089f..90f2b645aa0d 100644 --- a/arch/arm/lib/memcpy.S +++ b/arch/arm/lib/memcpy.S @@ -42,16 +42,13 @@ strb\cond \reg, [\ptr], #1 .endm - .macro enter reg1 reg2 - stmdb sp!, {r0, \reg1, \reg2} + .macro enter regs:vararg +UNWIND( .save {r0, \regs} ) + stmdb sp!, {r0, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 - ldmfd sp!, {r0, \reg1, \reg2} + .macro exit regs:vararg + ldmfd sp!, {r0, \regs} .endm .text diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 6fecc12a1f51..6410554039fd 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -31,12 +31,13 @@ WEAK(memmove) subs ip, r0, r1 cmphi r2, ip bls __memcpy - - stmfd sp!, {r0, r4, lr} UNWIND( .fnend ) UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ in first stmfd block + UNWIND( .save {r0, r4, fpreg, lr} ) + stmfd sp!, {r0, r4, UNWIND(fpreg,) lr} + UNWIND( .setfp fpreg, sp ) + UNWIND( mov fpreg, sp ) add r1, r1, r2 add r0, r0, r2 subs r2, r2, #4 @@ -48,12 +49,7 @@ WEAK(memmove) bne 10f 1: subs r2, r2, #(28) - stmfd sp!, {r5 - r8} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) - UNWIND( .save {r5 - r8} ) @ in second stmfd block + stmfd sp!, {r5, r6, r8, r9} blt 5f CALGN( ands ip, r0, #31 ) @@ -72,9 +68,9 @@ WEAK(memmove) PLD( pld [r1, #-96] ) 3: PLD( pld [r1, #-128] ) -4: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr} +4: ldmdb r1!, {r3, r4, r5, r6, r8, r9, ip, lr} subs r2, r2, #32 - stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr} + stmdb r0!, {r3, r4, r5, r6, r8, r9, ip, lr} bge 3b PLD( cmn r2, #96 ) PLD( bge 4b ) @@ -88,8 +84,8 @@ WEAK(memmove) W(ldr) r4, [r1, #-4]! W(ldr) r5, [r1, #-4]! W(ldr) r6, [r1, #-4]! - W(ldr) r7, [r1, #-4]! W(ldr) r8, [r1, #-4]! + W(ldr) r9, [r1, #-4]! W(ldr) lr, [r1, #-4]! add pc, pc, ip @@ -99,17 +95,13 @@ WEAK(memmove) W(str) r4, [r0, #-4]! W(str) r5, [r0, #-4]! W(str) r6, [r0, #-4]! - W(str) r7, [r0, #-4]! W(str) r8, [r0, #-4]! + W(str) r9, [r0, #-4]! W(str) lr, [r0, #-4]! CALGN( bcs 2b ) -7: ldmfd sp!, {r5 - r8} - UNWIND( .fnend ) @ end of second stmfd block - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block +7: ldmfd sp!, {r5, r6, r8, r9} 8: movs r2, r2, lsl #31 ldrbne r3, [r1, #-1]! @@ -118,7 +110,7 @@ WEAK(memmove) strbne r3, [r0, #-1]! strbcs r4, [r0, #-1]! strbcs ip, [r0, #-1] - ldmfd sp!, {r0, r4, pc} + ldmfd sp!, {r0, r4, UNWIND(fpreg,) pc} 9: cmp ip, #2 ldrbgt r3, [r1, #-1]! @@ -137,13 +129,10 @@ WEAK(memmove) ldr r3, [r1, #0] beq 17f blt 18f - UNWIND( .fnend ) .macro backward_copy_shift push pull - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block subs r2, r2, #28 blt 14f @@ -152,12 +141,7 @@ WEAK(memmove) CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) -11: stmfd sp!, {r5 - r9} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) - UNWIND( .save {r5 - r9} ) @ in new second stmfd block +11: stmfd sp!, {r5, r6, r8 - r10} PLD( pld [r1, #-4] ) PLD( subs r2, r2, #96 ) @@ -167,35 +151,31 @@ WEAK(memmove) PLD( pld [r1, #-96] ) 12: PLD( pld [r1, #-128] ) -13: ldmdb r1!, {r7, r8, r9, ip} +13: ldmdb r1!, {r8, r9, r10, ip} mov lr, r3, lspush #\push subs r2, r2, #32 ldmdb r1!, {r3, r4, r5, r6} orr lr, lr, ip, lspull #\pull mov ip, ip, lspush #\push - orr ip, ip, r9, lspull #\pull + orr ip, ip, r10, lspull #\pull + mov r10, r10, lspush #\push + orr r10, r10, r9, lspull #\pull mov r9, r9, lspush #\push orr r9, r9, r8, lspull #\pull mov r8, r8, lspush #\push - orr r8, r8, r7, lspull #\pull - mov r7, r7, lspush #\push - orr r7, r7, r6, lspull #\pull + orr r8, r8, r6, lspull #\pull mov r6, r6, lspush #\push orr r6, r6, r5, lspull #\pull mov r5, r5, lspush #\push orr r5, r5, r4, lspull #\pull mov r4, r4, lspush #\push orr r4, r4, r3, lspull #\pull - stmdb r0!, {r4 - r9, ip, lr} + stmdb r0!, {r4 - r6, r8 - r10, ip, lr} bge 12b PLD( cmn r2, #96 ) PLD( bge 13b ) - ldmfd sp!, {r5 - r9} - UNWIND( .fnend ) @ end of the second stmfd block - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block + ldmfd sp!, {r5, r6, r8 - r10} 14: ands ip, r2, #28 beq 16f @@ -211,7 +191,6 @@ WEAK(memmove) 16: add r1, r1, #(\pull / 8) b 8b - UNWIND( .fnend ) .endm @@ -222,5 +201,6 @@ WEAK(memmove) 18: backward_copy_shift push=24 pull=8 + UNWIND( .fnend ) ENDPROC(memmove) ENDPROC(__memmove) diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S index 9817cb258c1a..d71ab61430b2 100644 --- a/arch/arm/lib/memset.S +++ b/arch/arm/lib/memset.S @@ -28,16 +28,16 @@ UNWIND( .fnstart ) mov r3, r1 7: cmp r2, #16 blt 4f +UNWIND( .fnend ) #if ! CALGN(1)+0 /* * We need 2 extra registers for this loop - use r8 and the LR */ - stmfd sp!, {r8, lr} -UNWIND( .fnend ) UNWIND( .fnstart ) UNWIND( .save {r8, lr} ) + stmfd sp!, {r8, lr} mov r8, r1 mov lr, r3 @@ -66,10 +66,9 @@ UNWIND( .fnend ) * whole cache lines at once. */ - stmfd sp!, {r4-r8, lr} -UNWIND( .fnend ) UNWIND( .fnstart ) UNWIND( .save {r4-r8, lr} ) + stmfd sp!, {r4-r8, lr} mov r4, r1 mov r5, r3 mov r6, r1 diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 106f83a5ea6d..c30b689bec2e 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -92,11 +92,6 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) unsigned long ua_flags; int atomic; - if (uaccess_kernel()) { - memcpy((void *)to, from, n); - return 0; - } - /* the mmap semaphore is taken only if not in an atomic context */ atomic = faulthandler_disabled(); @@ -165,11 +160,6 @@ __clear_user_memset(void __user *addr, unsigned long n) { unsigned long ua_flags; - if (uaccess_kernel()) { - memset((void *)addr, 0, n); - return 0; - } - mmap_read_lock(current->mm); while (n) { pte_t *pte; diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c index b99dd8e1c93f..522510baed49 100644 --- a/arch/arm/lib/xor-neon.c +++ b/arch/arm/lib/xor-neon.c @@ -17,17 +17,11 @@ MODULE_LICENSE("GPL"); /* * Pull in the reference implementations while instructing GCC (through * -ftree-vectorize) to attempt to exploit implicit parallelism and emit - * NEON instructions. + * NEON instructions. Clang does this by default at O2 so no pragma is + * needed. */ -#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) +#ifdef CONFIG_CC_IS_GCC #pragma GCC optimize "tree-vectorize" -#else -/* - * While older versions of GCC do not generate incorrect code, they fail to - * recognize the parallel nature of these functions, and emit plain ARM code, - * which is known to be slower than the optimized ARM code in asm-arm/xor.h. - */ -#warning This code requires at least version 4.6 of GCC #endif #pragma GCC diagnostic ignored "-Wunused-variable" diff --git a/arch/arm/mach-bcm/Makefile b/arch/arm/mach-bcm/Makefile index 7baa8c9427d5..b2394ddb0558 100644 --- a/arch/arm/mach-bcm/Makefile +++ b/arch/arm/mach-bcm/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_ARCH_BCM_MOBILE_L2_CACHE) += kona_l2_cache.o # Support for secure monitor traps obj-$(CONFIG_ARCH_BCM_MOBILE_SMC) += bcm_kona_smc.o +CFLAGS_REMOVE_bcm_kona_smc.o += $(CC_FLAGS_FTRACE) # BCM2835 ifeq ($(CONFIG_ARCH_BCM2835),y) diff --git a/arch/arm/mach-exynos/mcpm-exynos.c b/arch/arm/mach-exynos/mcpm-exynos.c index cd861c57d5ad..fd0dbeb93357 100644 --- a/arch/arm/mach-exynos/mcpm-exynos.c +++ b/arch/arm/mach-exynos/mcpm-exynos.c @@ -35,7 +35,6 @@ static bool secure_firmware __ro_after_init; */ #define exynos_v7_exit_coherency_flush(level) \ asm volatile( \ - "stmfd sp!, {fp, ip}\n\t"\ "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \ "bic r0, r0, #"__stringify(CR_C)"\n\t" \ "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \ @@ -50,11 +49,10 @@ static bool secure_firmware __ro_after_init; "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \ "isb\n\t" \ "dsb\n\t" \ - "ldmfd sp!, {fp, ip}" \ : \ : "Ir" (pmu_base_addr + S5P_INFORM0) \ - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \ - "r9", "r10", "lr", "memory") + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", \ + "r9", "r10", "ip", "lr", "memory") static int exynos_cpu_powerup(unsigned int cpu, unsigned int cluster) { diff --git a/arch/arm/mach-footbridge/common.c b/arch/arm/mach-footbridge/common.c index eee095f0e2f6..322495df271d 100644 --- a/arch/arm/mach-footbridge/common.c +++ b/arch/arm/mach-footbridge/common.c @@ -27,6 +27,91 @@ #include "common.h" +#include <mach/hardware.h> +#include <mach/irqs.h> +#include <asm/hardware/dec21285.h> + +static int dc21285_get_irq(void) +{ + void __iomem *irqstatus = (void __iomem *)CSR_IRQ_STATUS; + u32 mask = readl(irqstatus); + + if (mask & IRQ_MASK_SDRAMPARITY) + return IRQ_SDRAMPARITY; + + if (mask & IRQ_MASK_UART_RX) + return IRQ_CONRX; + + if (mask & IRQ_MASK_DMA1) + return IRQ_DMA1; + + if (mask & IRQ_MASK_DMA2) + return IRQ_DMA2; + + if (mask & IRQ_MASK_IN0) + return IRQ_IN0; + + if (mask & IRQ_MASK_IN1) + return IRQ_IN1; + + if (mask & IRQ_MASK_IN2) + return IRQ_IN2; + + if (mask & IRQ_MASK_IN3) + return IRQ_IN3; + + if (mask & IRQ_MASK_PCI) + return IRQ_PCI; + + if (mask & IRQ_MASK_DOORBELLHOST) + return IRQ_DOORBELLHOST; + + if (mask & IRQ_MASK_I2OINPOST) + return IRQ_I2OINPOST; + + if (mask & IRQ_MASK_TIMER1) + return IRQ_TIMER1; + + if (mask & IRQ_MASK_TIMER2) + return IRQ_TIMER2; + + if (mask & IRQ_MASK_TIMER3) + return IRQ_TIMER3; + + if (mask & IRQ_MASK_UART_TX) + return IRQ_CONTX; + + if (mask & IRQ_MASK_PCI_ABORT) + return IRQ_PCI_ABORT; + + if (mask & IRQ_MASK_PCI_SERR) + return IRQ_PCI_SERR; + + if (mask & IRQ_MASK_DISCARD_TIMER) + return IRQ_DISCARD_TIMER; + + if (mask & IRQ_MASK_PCI_DPERR) + return IRQ_PCI_DPERR; + + if (mask & IRQ_MASK_PCI_PERR) + return IRQ_PCI_PERR; + + return 0; +} + +static void dc21285_handle_irq(struct pt_regs *regs) +{ + int irq; + do { + irq = dc21285_get_irq(); + if (!irq) + break; + + generic_handle_irq(irq); + } while (1); +} + + unsigned int mem_fclk_21285 = 50000000; EXPORT_SYMBOL(mem_fclk_21285); @@ -108,6 +193,8 @@ static void __init __fb_init_irq(void) void __init footbridge_init_irq(void) { + set_handle_irq(dc21285_handle_irq); + __fb_init_irq(); if (!footbridge_cfn_mode()) diff --git a/arch/arm/mach-footbridge/include/mach/entry-macro.S b/arch/arm/mach-footbridge/include/mach/entry-macro.S deleted file mode 100644 index dabbd5c54a78..000000000000 --- a/arch/arm/mach-footbridge/include/mach/entry-macro.S +++ /dev/null @@ -1,107 +0,0 @@ -/* - * arch/arm/mach-footbridge/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for footbridge-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ -#include <mach/hardware.h> -#include <mach/irqs.h> -#include <asm/hardware/dec21285.h> - - .equ dc21285_high, ARMCSR_BASE & 0xff000000 - .equ dc21285_low, ARMCSR_BASE & 0x00ffffff - - .macro get_irqnr_preamble, base, tmp - mov \base, #dc21285_high - .if dc21285_low - orr \base, \base, #dc21285_low - .endif - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldr \irqstat, [\base, #0x180] @ get interrupts - - mov \irqnr, #IRQ_SDRAMPARITY - tst \irqstat, #IRQ_MASK_SDRAMPARITY - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_RX - movne \irqnr, #IRQ_CONRX - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA1 - movne \irqnr, #IRQ_DMA1 - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA2 - movne \irqnr, #IRQ_DMA2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN0 - movne \irqnr, #IRQ_IN0 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN1 - movne \irqnr, #IRQ_IN1 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN2 - movne \irqnr, #IRQ_IN2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN3 - movne \irqnr, #IRQ_IN3 - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI - movne \irqnr, #IRQ_PCI - bne 1001f - - tst \irqstat, #IRQ_MASK_DOORBELLHOST - movne \irqnr, #IRQ_DOORBELLHOST - bne 1001f - - tst \irqstat, #IRQ_MASK_I2OINPOST - movne \irqnr, #IRQ_I2OINPOST - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER1 - movne \irqnr, #IRQ_TIMER1 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER2 - movne \irqnr, #IRQ_TIMER2 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER3 - movne \irqnr, #IRQ_TIMER3 - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_TX - movne \irqnr, #IRQ_CONTX - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_ABORT - movne \irqnr, #IRQ_PCI_ABORT - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_SERR - movne \irqnr, #IRQ_PCI_SERR - bne 1001f - - tst \irqstat, #IRQ_MASK_DISCARD_TIMER - movne \irqnr, #IRQ_DISCARD_TIMER - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_DPERR - movne \irqnr, #IRQ_PCI_DPERR - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_PERR - movne \irqnr, #IRQ_PCI_PERR -1001: - .endm - diff --git a/arch/arm/mach-iop32x/cp6.c b/arch/arm/mach-iop32x/cp6.c index ec74b07fb7e3..2882674a1c39 100644 --- a/arch/arm/mach-iop32x/cp6.c +++ b/arch/arm/mach-iop32x/cp6.c @@ -7,7 +7,7 @@ #include <asm/traps.h> #include <asm/ptrace.h> -static int cp6_trap(struct pt_regs *regs, unsigned int instr) +void iop_enable_cp6(void) { u32 temp; @@ -16,7 +16,15 @@ static int cp6_trap(struct pt_regs *regs, unsigned int instr) "mrc p15, 0, %0, c15, c1, 0\n\t" "orr %0, %0, #(1 << 6)\n\t" "mcr p15, 0, %0, c15, c1, 0\n\t" + "mrc p15, 0, %0, c15, c1, 0\n\t" + "mov %0, %0\n\t" + "sub pc, pc, #4 @ cp_wait\n\t" : "=r"(temp)); +} + +static int cp6_trap(struct pt_regs *regs, unsigned int instr) +{ + iop_enable_cp6(); return 0; } diff --git a/arch/arm/mach-iop32x/include/mach/entry-macro.S b/arch/arm/mach-iop32x/include/mach/entry-macro.S deleted file mode 100644 index 8e6766d4621e..000000000000 --- a/arch/arm/mach-iop32x/include/mach/entry-macro.S +++ /dev/null @@ -1,31 +0,0 @@ -/* - * arch/arm/mach-iop32x/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for IOP32x-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - .macro get_irqnr_preamble, base, tmp - mrc p15, 0, \tmp, c15, c1, 0 - orr \tmp, \tmp, #(1 << 6) - mcr p15, 0, \tmp, c15, c1, 0 @ Enable cp6 access - mrc p15, 0, \tmp, c15, c1, 0 - mov \tmp, \tmp - sub pc, pc, #4 @ cp_wait - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - mrc p6, 0, \irqstat, c8, c0, 0 @ Read IINTSRC - cmp \irqstat, #0 - clzne \irqnr, \irqstat - rsbne \irqnr, \irqnr, #31 - .endm - - .macro arch_ret_to_user, tmp1, tmp2 - mrc p15, 0, \tmp1, c15, c1, 0 - ands \tmp2, \tmp1, #(1 << 6) - bicne \tmp1, \tmp1, #(1 << 6) - mcrne p15, 0, \tmp1, c15, c1, 0 @ Disable cp6 access - .endm diff --git a/arch/arm/mach-iop32x/include/mach/irqs.h b/arch/arm/mach-iop32x/include/mach/irqs.h index c4e78df428e8..e09ae5f48aec 100644 --- a/arch/arm/mach-iop32x/include/mach/irqs.h +++ b/arch/arm/mach-iop32x/include/mach/irqs.h @@ -9,6 +9,6 @@ #ifndef __IRQS_H #define __IRQS_H -#define NR_IRQS 32 +#define NR_IRQS 33 #endif diff --git a/arch/arm/mach-iop32x/iop3xx.h b/arch/arm/mach-iop32x/iop3xx.h index 46b4b34a4ad2..a6ec7ebadb35 100644 --- a/arch/arm/mach-iop32x/iop3xx.h +++ b/arch/arm/mach-iop32x/iop3xx.h @@ -225,6 +225,7 @@ extern int iop3xx_get_init_atu(void); #include <linux/reboot.h> void iop3xx_map_io(void); +void iop_enable_cp6(void); void iop_init_cp6_handler(void); void iop_init_time(unsigned long tickrate); void iop3xx_restart(enum reboot_mode, const char *); diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c index 2d48bf1398c1..6dca7e97d81f 100644 --- a/arch/arm/mach-iop32x/irq.c +++ b/arch/arm/mach-iop32x/irq.c @@ -29,17 +29,26 @@ static void intstr_write(u32 val) asm volatile("mcr p6, 0, %0, c4, c0, 0" : : "r" (val)); } +static u32 iintsrc_read(void) +{ + int irq; + + asm volatile("mrc p6, 0, %0, c8, c0, 0" : "=r" (irq)); + + return irq; +} + static void iop32x_irq_mask(struct irq_data *d) { - iop32x_mask &= ~(1 << d->irq); + iop32x_mask &= ~(1 << (d->irq - 1)); intctl_write(iop32x_mask); } static void iop32x_irq_unmask(struct irq_data *d) { - iop32x_mask |= 1 << d->irq; + iop32x_mask |= 1 << (d->irq - 1); intctl_write(iop32x_mask); } @@ -50,11 +59,25 @@ struct irq_chip ext_chip = { .irq_unmask = iop32x_irq_unmask, }; +static void iop_handle_irq(struct pt_regs *regs) +{ + u32 mask; + + iop_enable_cp6(); + + do { + mask = iintsrc_read(); + if (mask) + generic_handle_irq(fls(mask)); + } while (mask); +} + void __init iop32x_init_irq(void) { int i; iop_init_cp6_handler(); + set_handle_irq(iop_handle_irq); intctl_write(0); intstr_write(0); @@ -65,7 +88,7 @@ void __init iop32x_init_irq(void) machine_is_em7210()) *IOP3XX_PCIIRSR = 0x0f; - for (i = 0; i < NR_IRQS; i++) { + for (i = 1; i < NR_IRQS; i++) { irq_set_chip_and_handler(i, &ext_chip, handle_level_irq); irq_clear_status_flags(i, IRQ_NOREQUEST | IRQ_NOPROBE); } diff --git a/arch/arm/mach-iop32x/irqs.h b/arch/arm/mach-iop32x/irqs.h index 69858e4e905d..e1dfc8b4e7d7 100644 --- a/arch/arm/mach-iop32x/irqs.h +++ b/arch/arm/mach-iop32x/irqs.h @@ -7,36 +7,40 @@ #ifndef __IOP32X_IRQS_H #define __IOP32X_IRQS_H +/* Interrupts in Linux start at 1, hardware starts at 0 */ + +#define IOP_IRQ(x) ((x) + 1) + /* * IOP80321 chipset interrupts */ -#define IRQ_IOP32X_DMA0_EOT 0 -#define IRQ_IOP32X_DMA0_EOC 1 -#define IRQ_IOP32X_DMA1_EOT 2 -#define IRQ_IOP32X_DMA1_EOC 3 -#define IRQ_IOP32X_AA_EOT 6 -#define IRQ_IOP32X_AA_EOC 7 -#define IRQ_IOP32X_CORE_PMON 8 -#define IRQ_IOP32X_TIMER0 9 -#define IRQ_IOP32X_TIMER1 10 -#define IRQ_IOP32X_I2C_0 11 -#define IRQ_IOP32X_I2C_1 12 -#define IRQ_IOP32X_MESSAGING 13 -#define IRQ_IOP32X_ATU_BIST 14 -#define IRQ_IOP32X_PERFMON 15 -#define IRQ_IOP32X_CORE_PMU 16 -#define IRQ_IOP32X_BIU_ERR 17 -#define IRQ_IOP32X_ATU_ERR 18 -#define IRQ_IOP32X_MCU_ERR 19 -#define IRQ_IOP32X_DMA0_ERR 20 -#define IRQ_IOP32X_DMA1_ERR 21 -#define IRQ_IOP32X_AA_ERR 23 -#define IRQ_IOP32X_MSG_ERR 24 -#define IRQ_IOP32X_SSP 25 -#define IRQ_IOP32X_XINT0 27 -#define IRQ_IOP32X_XINT1 28 -#define IRQ_IOP32X_XINT2 29 -#define IRQ_IOP32X_XINT3 30 -#define IRQ_IOP32X_HPI 31 +#define IRQ_IOP32X_DMA0_EOT IOP_IRQ(0) +#define IRQ_IOP32X_DMA0_EOC IOP_IRQ(1) +#define IRQ_IOP32X_DMA1_EOT IOP_IRQ(2) +#define IRQ_IOP32X_DMA1_EOC IOP_IRQ(3) +#define IRQ_IOP32X_AA_EOT IOP_IRQ(6) +#define IRQ_IOP32X_AA_EOC IOP_IRQ(7) +#define IRQ_IOP32X_CORE_PMON IOP_IRQ(8) +#define IRQ_IOP32X_TIMER0 IOP_IRQ(9) +#define IRQ_IOP32X_TIMER1 IOP_IRQ(10) +#define IRQ_IOP32X_I2C_0 IOP_IRQ(11) +#define IRQ_IOP32X_I2C_1 IOP_IRQ(12) +#define IRQ_IOP32X_MESSAGING IOP_IRQ(13) +#define IRQ_IOP32X_ATU_BIST IOP_IRQ(14) +#define IRQ_IOP32X_PERFMON IOP_IRQ(15) +#define IRQ_IOP32X_CORE_PMU IOP_IRQ(16) +#define IRQ_IOP32X_BIU_ERR IOP_IRQ(17) +#define IRQ_IOP32X_ATU_ERR IOP_IRQ(18) +#define IRQ_IOP32X_MCU_ERR IOP_IRQ(19) +#define IRQ_IOP32X_DMA0_ERR IOP_IRQ(20) +#define IRQ_IOP32X_DMA1_ERR IOP_IRQ(21) +#define IRQ_IOP32X_AA_ERR IOP_IRQ(23) +#define IRQ_IOP32X_MSG_ERR IOP_IRQ(24) +#define IRQ_IOP32X_SSP IOP_IRQ(25) +#define IRQ_IOP32X_XINT0 IOP_IRQ(27) +#define IRQ_IOP32X_XINT1 IOP_IRQ(28) +#define IRQ_IOP32X_XINT2 IOP_IRQ(29) +#define IRQ_IOP32X_XINT3 IOP_IRQ(30) +#define IRQ_IOP32X_HPI IOP_IRQ(31) #endif diff --git a/arch/arm/mach-mstar/Kconfig b/arch/arm/mach-mstar/Kconfig index cd300eeedc20..0bf4d312bcfd 100644 --- a/arch/arm/mach-mstar/Kconfig +++ b/arch/arm/mach-mstar/Kconfig @@ -3,6 +3,7 @@ menuconfig ARCH_MSTARV7 depends on ARCH_MULTI_V7 select ARM_GIC select ARM_HEAVY_MB + select HAVE_ARM_ARCH_TIMER select MST_IRQ select MSTAR_MSC313_MPLL help diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c index 6daaa645ae5d..21413a9b7b6c 100644 --- a/arch/arm/mach-omap2/display.c +++ b/arch/arm/mach-omap2/display.c @@ -263,9 +263,9 @@ static int __init omapdss_init_of(void) } r = of_platform_populate(node, NULL, NULL, &pdev->dev); + put_device(&pdev->dev); if (r) { pr_err("Unable to populate DSS submodule devices\n"); - put_device(&pdev->dev); return r; } diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c index ccb0e3732c0d..31d1a21f6041 100644 --- a/arch/arm/mach-omap2/omap_hwmod.c +++ b/arch/arm/mach-omap2/omap_hwmod.c @@ -752,8 +752,10 @@ static int __init _init_clkctrl_providers(void) for_each_matching_node(np, ti_clkctrl_match_table) { ret = _setup_clkctrl_provider(np); - if (ret) + if (ret) { + of_node_put(np); break; + } } return ret; diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c index 593c7f793da5..44659fbc37ba 100644 --- a/arch/arm/mach-pxa/corgi.c +++ b/arch/arm/mach-pxa/corgi.c @@ -530,6 +530,16 @@ static struct pxa2xx_spi_controller corgi_spi_info = { .num_chipselect = 3, }; +static struct gpiod_lookup_table corgi_spi_gpio_table = { + .dev_id = "pxa2xx-spi.1", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", CORGI_GPIO_ADS7846_CS, "cs", 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", CORGI_GPIO_LCDCON_CS, "cs", 1, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", CORGI_GPIO_MAX1111_CS, "cs", 2, GPIO_ACTIVE_LOW), + { }, + }, +}; + static void corgi_wait_for_hsync(void) { while (gpio_get_value(CORGI_GPIO_HSYNC)) @@ -548,10 +558,6 @@ static struct ads7846_platform_data corgi_ads7846_info = { .wait_for_sync = corgi_wait_for_hsync, }; -static struct pxa2xx_spi_chip corgi_ads7846_chip = { - .gpio_cs = CORGI_GPIO_ADS7846_CS, -}; - static void corgi_bl_kick_battery(void) { void (*kick_batt)(void); @@ -580,14 +586,6 @@ static struct corgi_lcd_platform_data corgi_lcdcon_info = { .kick_battery = corgi_bl_kick_battery, }; -static struct pxa2xx_spi_chip corgi_lcdcon_chip = { - .gpio_cs = CORGI_GPIO_LCDCON_CS, -}; - -static struct pxa2xx_spi_chip corgi_max1111_chip = { - .gpio_cs = CORGI_GPIO_MAX1111_CS, -}; - static struct spi_board_info corgi_spi_devices[] = { { .modalias = "ads7846", @@ -595,7 +593,6 @@ static struct spi_board_info corgi_spi_devices[] = { .bus_num = 1, .chip_select = 0, .platform_data = &corgi_ads7846_info, - .controller_data= &corgi_ads7846_chip, .irq = PXA_GPIO_TO_IRQ(CORGI_GPIO_TP_INT), }, { .modalias = "corgi-lcd", @@ -603,18 +600,17 @@ static struct spi_board_info corgi_spi_devices[] = { .bus_num = 1, .chip_select = 1, .platform_data = &corgi_lcdcon_info, - .controller_data= &corgi_lcdcon_chip, }, { .modalias = "max1111", .max_speed_hz = 450000, .bus_num = 1, .chip_select = 2, - .controller_data= &corgi_max1111_chip, }, }; static void __init corgi_init_spi(void) { + gpiod_add_lookup_table(&corgi_spi_gpio_table); pxa2xx_set_spi_info(1, &corgi_spi_info); gpiod_add_lookup_table(&corgi_lcdcon_gpio_table); spi_register_board_info(ARRAY_AND_SIZE(corgi_spi_devices)); diff --git a/arch/arm/mach-pxa/hx4700.c b/arch/arm/mach-pxa/hx4700.c index 1d4c5db54be2..e1870fbb19e7 100644 --- a/arch/arm/mach-pxa/hx4700.c +++ b/arch/arm/mach-pxa/hx4700.c @@ -616,7 +616,6 @@ static struct pxa2xx_spi_chip tsc2046_chip = { .tx_threshold = 1, .rx_threshold = 2, .timeout = 64, - .gpio_cs = GPIO88_HX4700_TSC2046_CS, }; static struct spi_board_info tsc2046_board_info[] __initdata = { @@ -635,6 +634,14 @@ static struct pxa2xx_spi_controller pxa_ssp2_master_info = { .enable_dma = 1, }; +static struct gpiod_lookup_table pxa_ssp2_gpio_table = { + .dev_id = "pxa2xx-spi.2", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", GPIO88_HX4700_TSC2046_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + /* * External power */ @@ -896,6 +903,7 @@ static void __init hx4700_init(void) pxa_set_i2c_info(NULL); i2c_register_board_info(0, ARRAY_AND_SIZE(i2c_board_info)); i2c_register_board_info(1, ARRAY_AND_SIZE(pi2c_board_info)); + gpiod_add_lookup_table(&pxa_ssp2_gpio_table); pxa2xx_set_spi_info(2, &pxa_ssp2_master_info); spi_register_board_info(ARRAY_AND_SIZE(tsc2046_board_info)); diff --git a/arch/arm/mach-pxa/icontrol.c b/arch/arm/mach-pxa/icontrol.c index 04a12523cdee..753fe166ab68 100644 --- a/arch/arm/mach-pxa/icontrol.c +++ b/arch/arm/mach-pxa/icontrol.c @@ -13,7 +13,7 @@ #include <linux/irq.h> #include <linux/platform_device.h> #include <linux/property.h> -#include <linux/gpio.h> +#include <linux/gpio/machine.h> #include <asm/mach-types.h> #include <asm/mach/arch.h> @@ -42,7 +42,6 @@ static struct pxa2xx_spi_chip mcp251x_chip_info1 = { .rx_threshold = 128, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = ICONTROL_MCP251x_nCS1 }; static struct pxa2xx_spi_chip mcp251x_chip_info2 = { @@ -50,7 +49,6 @@ static struct pxa2xx_spi_chip mcp251x_chip_info2 = { .rx_threshold = 128, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = ICONTROL_MCP251x_nCS2 }; static struct pxa2xx_spi_chip mcp251x_chip_info3 = { @@ -58,7 +56,6 @@ static struct pxa2xx_spi_chip mcp251x_chip_info3 = { .rx_threshold = 128, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = ICONTROL_MCP251x_nCS3 }; static struct pxa2xx_spi_chip mcp251x_chip_info4 = { @@ -66,7 +63,6 @@ static struct pxa2xx_spi_chip mcp251x_chip_info4 = { .rx_threshold = 128, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = ICONTROL_MCP251x_nCS4 }; static const struct property_entry mcp251x_properties[] = { @@ -143,6 +139,24 @@ struct platform_device pxa_spi_ssp4 = { } }; +static struct gpiod_lookup_table pxa_ssp3_gpio_table = { + .dev_id = "pxa2xx-spi.3", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", ICONTROL_MCP251x_nCS1, "cs", 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", ICONTROL_MCP251x_nCS2, "cs", 1, GPIO_ACTIVE_LOW), + { }, + }, +}; + +static struct gpiod_lookup_table pxa_ssp4_gpio_table = { + .dev_id = "pxa2xx-spi.4", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", ICONTROL_MCP251x_nCS3, "cs", 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", ICONTROL_MCP251x_nCS4, "cs", 1, GPIO_ACTIVE_LOW), + { }, + }, +}; + static struct platform_device *icontrol_spi_devices[] __initdata = { &pxa_spi_ssp3, &pxa_spi_ssp4, @@ -175,6 +189,8 @@ static mfp_cfg_t mfp_can_cfg[] __initdata = { static void __init icontrol_can_init(void) { pxa3xx_mfp_config(ARRAY_AND_SIZE(mfp_can_cfg)); + gpiod_add_lookup_table(&pxa_ssp3_gpio_table); + gpiod_add_lookup_table(&pxa_ssp4_gpio_table); platform_add_devices(ARRAY_AND_SIZE(icontrol_spi_devices)); spi_register_board_info(ARRAY_AND_SIZE(mcp251x_board_info)); } diff --git a/arch/arm/mach-pxa/littleton.c b/arch/arm/mach-pxa/littleton.c index 793f61375ee8..73f5953b3bb6 100644 --- a/arch/arm/mach-pxa/littleton.c +++ b/arch/arm/mach-pxa/littleton.c @@ -195,7 +195,6 @@ static struct pxa2xx_spi_controller littleton_spi_info = { static struct pxa2xx_spi_chip littleton_tdo24m_chip = { .rx_threshold = 1, .tx_threshold = 1, - .gpio_cs = LITTLETON_GPIO_LCD_CS, }; static struct spi_board_info littleton_spi_devices[] __initdata = { @@ -208,8 +207,17 @@ static struct spi_board_info littleton_spi_devices[] __initdata = { }, }; +static struct gpiod_lookup_table littleton_spi_gpio_table = { + .dev_id = "pxa2xx-spi.2", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", LITTLETON_GPIO_LCD_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + static void __init littleton_init_spi(void) { + gpiod_add_lookup_table(&littleton_spi_gpio_table); pxa2xx_set_spi_info(2, &littleton_spi_info); spi_register_board_info(ARRAY_AND_SIZE(littleton_spi_devices)); } diff --git a/arch/arm/mach-pxa/magician.c b/arch/arm/mach-pxa/magician.c index cd9fa465b9b2..200fd35168e0 100644 --- a/arch/arm/mach-pxa/magician.c +++ b/arch/arm/mach-pxa/magician.c @@ -938,8 +938,6 @@ struct pxa2xx_spi_chip tsc2046_chip_info = { .tx_threshold = 1, .rx_threshold = 2, .timeout = 64, - /* NOTICE must be GPIO, incompatibility with hw PXA SPI framing */ - .gpio_cs = GPIO14_MAGICIAN_TSC2046_CS, }; static struct pxa2xx_spi_controller magician_spi_info = { @@ -947,6 +945,15 @@ static struct pxa2xx_spi_controller magician_spi_info = { .enable_dma = 1, }; +static struct gpiod_lookup_table magician_spi_gpio_table = { + .dev_id = "pxa2xx-spi.2", + .table = { + /* NOTICE must be GPIO, incompatibility with hw PXA SPI framing */ + GPIO_LOOKUP_IDX("gpio-pxa", GPIO14_MAGICIAN_TSC2046_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + static struct spi_board_info ads7846_spi_board_info[] __initdata = { { .modalias = "ads7846", @@ -1031,6 +1038,7 @@ static void __init magician_init(void) } else pr_err("LCD detection: CPLD mapping failed\n"); + gpiod_add_lookup_table(&magician_spi_gpio_table); pxa2xx_set_spi_info(2, &magician_spi_info); spi_register_board_info(ARRAY_AND_SIZE(ads7846_spi_board_info)); diff --git a/arch/arm/mach-pxa/poodle.c b/arch/arm/mach-pxa/poodle.c index 3a4ecc3c8f8b..58cfa434afde 100644 --- a/arch/arm/mach-pxa/poodle.c +++ b/arch/arm/mach-pxa/poodle.c @@ -197,6 +197,14 @@ static struct pxa2xx_spi_controller poodle_spi_info = { .num_chipselect = 1, }; +static struct gpiod_lookup_table poodle_spi_gpio_table = { + .dev_id = "pxa2xx-spi.1", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", POODLE_GPIO_TP_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + static struct ads7846_platform_data poodle_ads7846_info = { .model = 7846, .vref_delay_usecs = 100, @@ -205,23 +213,19 @@ static struct ads7846_platform_data poodle_ads7846_info = { .gpio_pendown = POODLE_GPIO_TP_INT, }; -static struct pxa2xx_spi_chip poodle_ads7846_chip = { - .gpio_cs = POODLE_GPIO_TP_CS, -}; - static struct spi_board_info poodle_spi_devices[] = { { .modalias = "ads7846", .max_speed_hz = 10000, .bus_num = 1, .platform_data = &poodle_ads7846_info, - .controller_data= &poodle_ads7846_chip, .irq = PXA_GPIO_TO_IRQ(POODLE_GPIO_TP_INT), }, }; static void __init poodle_init_spi(void) { + gpiod_add_lookup_table(&poodle_spi_gpio_table); pxa2xx_set_spi_info(1, &poodle_spi_info); spi_register_board_info(ARRAY_AND_SIZE(poodle_spi_devices)); } diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c index 371008e9bb02..a648e7094e84 100644 --- a/arch/arm/mach-pxa/spitz.c +++ b/arch/arm/mach-pxa/spitz.c @@ -510,10 +510,6 @@ static struct ads7846_platform_data spitz_ads7846_info = { .wait_for_sync = spitz_ads7846_wait_for_hsync, }; -static struct pxa2xx_spi_chip spitz_ads7846_chip = { - .gpio_cs = SPITZ_GPIO_ADS7846_CS, -}; - static void spitz_bl_kick_battery(void) { void (*kick_batt)(void); @@ -555,14 +551,6 @@ static struct corgi_lcd_platform_data spitz_lcdcon_info = { .kick_battery = spitz_bl_kick_battery, }; -static struct pxa2xx_spi_chip spitz_lcdcon_chip = { - .gpio_cs = SPITZ_GPIO_LCDCON_CS, -}; - -static struct pxa2xx_spi_chip spitz_max1111_chip = { - .gpio_cs = SPITZ_GPIO_MAX1111_CS, -}; - static struct spi_board_info spitz_spi_devices[] = { { .modalias = "ads7846", @@ -570,7 +558,6 @@ static struct spi_board_info spitz_spi_devices[] = { .bus_num = 2, .chip_select = 0, .platform_data = &spitz_ads7846_info, - .controller_data = &spitz_ads7846_chip, .irq = PXA_GPIO_TO_IRQ(SPITZ_GPIO_TP_INT), }, { .modalias = "corgi-lcd", @@ -578,13 +565,11 @@ static struct spi_board_info spitz_spi_devices[] = { .bus_num = 2, .chip_select = 1, .platform_data = &spitz_lcdcon_info, - .controller_data = &spitz_lcdcon_chip, }, { .modalias = "max1111", .max_speed_hz = 450000, .bus_num = 2, .chip_select = 2, - .controller_data = &spitz_max1111_chip, }, }; @@ -592,6 +577,16 @@ static struct pxa2xx_spi_controller spitz_spi_info = { .num_chipselect = 3, }; +static struct gpiod_lookup_table spitz_spi_gpio_table = { + .dev_id = "pxa2xx-spi.2", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", SPITZ_GPIO_ADS7846_CS, "cs", 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", SPITZ_GPIO_LCDCON_CS, "cs", 1, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-pxa", SPITZ_GPIO_MAX1111_CS, "cs", 2, GPIO_ACTIVE_LOW), + { }, + }, +}; + static void __init spitz_spi_init(void) { if (machine_is_akita()) @@ -599,6 +594,7 @@ static void __init spitz_spi_init(void) else gpiod_add_lookup_table(&spitz_lcdcon_gpio_table); + gpiod_add_lookup_table(&spitz_spi_gpio_table); pxa2xx_set_spi_info(2, &spitz_spi_info); spi_register_board_info(ARRAY_AND_SIZE(spitz_spi_devices)); } diff --git a/arch/arm/mach-pxa/stargate2.c b/arch/arm/mach-pxa/stargate2.c index 8ca02ec1d44c..b43e2f4536a5 100644 --- a/arch/arm/mach-pxa/stargate2.c +++ b/arch/arm/mach-pxa/stargate2.c @@ -346,6 +346,22 @@ static struct pxa2xx_spi_controller pxa_ssp_master_2_info = { .num_chipselect = 1, }; +static struct gpiod_lookup_table pxa_ssp1_gpio_table = { + .dev_id = "pxa2xx-spi.1", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", 24, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + +static struct gpiod_lookup_table pxa_ssp3_gpio_table = { + .dev_id = "pxa2xx-spi.3", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", 39, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + /* An upcoming kernel change will scrap SFRM usage so these * drivers have been moved to use GPIOs */ static struct pxa2xx_spi_chip staccel_chip_info = { @@ -353,7 +369,6 @@ static struct pxa2xx_spi_chip staccel_chip_info = { .rx_threshold = 8, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = 24, }; static struct pxa2xx_spi_chip cc2420_info = { @@ -361,7 +376,6 @@ static struct pxa2xx_spi_chip cc2420_info = { .rx_threshold = 8, .dma_burst_size = 8, .timeout = 235, - .gpio_cs = 39, }; static struct spi_board_info spi_board_info[] __initdata = { @@ -410,6 +424,8 @@ static void __init imote2_stargate2_init(void) pxa_set_btuart_info(NULL); pxa_set_stuart_info(NULL); + gpiod_add_lookup_table(&pxa_ssp1_gpio_table); + gpiod_add_lookup_table(&pxa_ssp3_gpio_table); pxa2xx_set_spi_info(1, &pxa_ssp_master_0_info); pxa2xx_set_spi_info(2, &pxa_ssp_master_1_info); pxa2xx_set_spi_info(3, &pxa_ssp_master_2_info); diff --git a/arch/arm/mach-pxa/z2.c b/arch/arm/mach-pxa/z2.c index 8e74fbb0a96e..7eaeda269927 100644 --- a/arch/arm/mach-pxa/z2.c +++ b/arch/arm/mach-pxa/z2.c @@ -570,7 +570,6 @@ static struct pxa2xx_spi_chip z2_lbs_chip_info = { .rx_threshold = 8, .tx_threshold = 8, .timeout = 1000, - .gpio_cs = GPIO24_ZIPITZ2_WIFI_CS, }; static struct libertas_spi_platform_data z2_lbs_pdata = { @@ -584,7 +583,6 @@ static struct pxa2xx_spi_chip lms283_chip_info = { .rx_threshold = 1, .tx_threshold = 1, .timeout = 64, - .gpio_cs = GPIO88_ZIPITZ2_LCD_CS, }; static struct gpiod_lookup_table lms283_gpio_table = { @@ -624,8 +622,26 @@ static struct pxa2xx_spi_controller pxa_ssp2_master_info = { .num_chipselect = 1, }; +static struct gpiod_lookup_table pxa_ssp1_gpio_table = { + .dev_id = "pxa2xx-spi.1", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", GPIO24_ZIPITZ2_WIFI_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + +static struct gpiod_lookup_table pxa_ssp2_gpio_table = { + .dev_id = "pxa2xx-spi.2", + .table = { + GPIO_LOOKUP_IDX("gpio-pxa", GPIO88_ZIPITZ2_LCD_CS, "cs", 0, GPIO_ACTIVE_LOW), + { }, + }, +}; + static void __init z2_spi_init(void) { + gpiod_add_lookup_table(&pxa_ssp1_gpio_table); + gpiod_add_lookup_table(&pxa_ssp2_gpio_table); pxa2xx_set_spi_info(1, &pxa_ssp1_master_info); pxa2xx_set_spi_info(2, &pxa_ssp2_master_info); gpiod_add_lookup_table(&lms283_gpio_table); diff --git a/arch/arm/mach-rpc/fiq.S b/arch/arm/mach-rpc/fiq.S index 0de83e9b0b39..087bdf4bc093 100644 --- a/arch/arm/mach-rpc/fiq.S +++ b/arch/arm/mach-rpc/fiq.S @@ -2,10 +2,11 @@ #include <linux/linkage.h> #include <asm/assembler.h> #include <mach/hardware.h> -#include <mach/entry-macro.S> - .text + .equ ioc_base_high, IOC_BASE & 0xff000000 + .equ ioc_base_low, IOC_BASE & 0x00ff0000 + .text .global rpc_default_fiq_end ENTRY(rpc_default_fiq_start) mov r12, #ioc_base_high diff --git a/arch/arm/mach-rpc/include/mach/entry-macro.S b/arch/arm/mach-rpc/include/mach/entry-macro.S deleted file mode 100644 index a6d1a9f4bb79..000000000000 --- a/arch/arm/mach-rpc/include/mach/entry-macro.S +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include <mach/hardware.h> -#include <asm/hardware/entry-macro-iomd.S> - - .equ ioc_base_high, IOC_BASE & 0xff000000 - .equ ioc_base_low, IOC_BASE & 0x00ff0000 - - .macro get_irqnr_preamble, base, tmp - mov \base, #ioc_base_high @ point at IOC - .if ioc_base_low - orr \base, \base, #ioc_base_low - .endif - .endm diff --git a/arch/arm/mach-rpc/irq.c b/arch/arm/mach-rpc/irq.c index 803aeb126f0e..dc29384b6ef8 100644 --- a/arch/arm/mach-rpc/irq.c +++ b/arch/arm/mach-rpc/irq.c @@ -14,6 +14,99 @@ #define CLR 0x04 #define MASK 0x08 +static const u8 irq_prio_h[256] = { + 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, +}; + +static const u8 irq_prio_d[256] = { + 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, +}; + +static const u8 irq_prio_l[256] = { + 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +}; + +static int iomd_get_irq_nr(void) +{ + int irq; + u8 reg; + + /* get highest priority first */ + reg = readb(IOC_BASE + IOMD_IRQREQB); + irq = irq_prio_h[reg]; + if (irq) + return irq; + + /* get DMA */ + reg = readb(IOC_BASE + IOMD_DMAREQ); + irq = irq_prio_d[reg]; + if (irq) + return irq; + + /* get low priority */ + reg = readb(IOC_BASE + IOMD_IRQREQA); + irq = irq_prio_l[reg]; + if (irq) + return irq; + return 0; +} + +static void iomd_handle_irq(struct pt_regs *regs) +{ + int irq; + + do { + irq = iomd_get_irq_nr(); + if (irq) + generic_handle_irq(irq); + } while (irq); +} + static void __iomem *iomd_get_base(struct irq_data *d) { void *cd = irq_data_get_irq_chip_data(d); @@ -82,6 +175,8 @@ void __init rpc_init_irq(void) set_fiq_handler(&rpc_default_fiq_start, &rpc_default_fiq_end - &rpc_default_fiq_start); + set_handle_irq(iomd_handle_irq); + for (irq = 0; irq < NR_IRQS; irq++) { clr = IRQ_NOREQUEST; set = 0; diff --git a/arch/arm/mach-s3c/Kconfig b/arch/arm/mach-s3c/Kconfig index 25606e668cf9..1899fc3f44fd 100644 --- a/arch/arm/mach-s3c/Kconfig +++ b/arch/arm/mach-s3c/Kconfig @@ -191,18 +191,6 @@ config S3C64XX_DEV_SPI0 Compile in platform device definitions for S3C64XX's type SPI controller 0 -config S3C64XX_DEV_SPI1 - bool - help - Compile in platform device definitions for S3C64XX's type - SPI controller 1 - -config S3C64XX_DEV_SPI2 - bool - help - Compile in platform device definitions for S3C64XX's type - SPI controller 2 - config SAMSUNG_DEV_TS bool help diff --git a/arch/arm/mach-s3c/devs.c b/arch/arm/mach-s3c/devs.c index 06dec64848f9..1e266fc24f9b 100644 --- a/arch/arm/mach-s3c/devs.c +++ b/arch/arm/mach-s3c/devs.c @@ -1107,8 +1107,7 @@ struct platform_device s3c64xx_device_spi0 = { }, }; -void __init s3c64xx_spi0_set_platdata(int (*cfg_gpio)(void), int src_clk_nr, - int num_cs) +void __init s3c64xx_spi0_set_platdata(int src_clk_nr, int num_cs) { struct s3c64xx_spi_info pd; @@ -1120,80 +1119,8 @@ void __init s3c64xx_spi0_set_platdata(int (*cfg_gpio)(void), int src_clk_nr, pd.num_cs = num_cs; pd.src_clk_nr = src_clk_nr; - pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi0_cfg_gpio; + pd.cfg_gpio = s3c64xx_spi0_cfg_gpio; s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi0); } #endif /* CONFIG_S3C64XX_DEV_SPI0 */ - -#ifdef CONFIG_S3C64XX_DEV_SPI1 -static struct resource s3c64xx_spi1_resource[] = { - [0] = DEFINE_RES_MEM(S3C_PA_SPI1, SZ_256), - [1] = DEFINE_RES_IRQ(IRQ_SPI1), -}; - -struct platform_device s3c64xx_device_spi1 = { - .name = "s3c6410-spi", - .id = 1, - .num_resources = ARRAY_SIZE(s3c64xx_spi1_resource), - .resource = s3c64xx_spi1_resource, - .dev = { - .dma_mask = &samsung_device_dma_mask, - .coherent_dma_mask = DMA_BIT_MASK(32), - }, -}; - -void __init s3c64xx_spi1_set_platdata(int (*cfg_gpio)(void), int src_clk_nr, - int num_cs) -{ - struct s3c64xx_spi_info pd; - - /* Reject invalid configuration */ - if (!num_cs || src_clk_nr < 0) { - pr_err("%s: Invalid SPI configuration\n", __func__); - return; - } - - pd.num_cs = num_cs; - pd.src_clk_nr = src_clk_nr; - pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi1_cfg_gpio; - - s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi1); -} -#endif /* CONFIG_S3C64XX_DEV_SPI1 */ - -#ifdef CONFIG_S3C64XX_DEV_SPI2 -static struct resource s3c64xx_spi2_resource[] = { - [0] = DEFINE_RES_MEM(S3C_PA_SPI2, SZ_256), - [1] = DEFINE_RES_IRQ(IRQ_SPI2), -}; - -struct platform_device s3c64xx_device_spi2 = { - .name = "s3c6410-spi", - .id = 2, - .num_resources = ARRAY_SIZE(s3c64xx_spi2_resource), - .resource = s3c64xx_spi2_resource, - .dev = { - .dma_mask = &samsung_device_dma_mask, - .coherent_dma_mask = DMA_BIT_MASK(32), - }, -}; - -void __init s3c64xx_spi2_set_platdata(int (*cfg_gpio)(void), int src_clk_nr, - int num_cs) -{ - struct s3c64xx_spi_info pd; - - /* Reject invalid configuration */ - if (!num_cs || src_clk_nr < 0) { - pr_err("%s: Invalid SPI configuration\n", __func__); - return; - } - - pd.num_cs = num_cs; - pd.src_clk_nr = src_clk_nr; - pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi2_cfg_gpio; - - s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi2); -} -#endif /* CONFIG_S3C64XX_DEV_SPI2 */ diff --git a/arch/arm/mach-s3c/mach-crag6410-module.c b/arch/arm/mach-s3c/mach-crag6410-module.c index 407ad493493e..5d1d4b67a4b7 100644 --- a/arch/arm/mach-s3c/mach-crag6410-module.c +++ b/arch/arm/mach-s3c/mach-crag6410-module.c @@ -32,10 +32,6 @@ #include "crag6410.h" -static struct s3c64xx_spi_csinfo wm0010_spi_csinfo = { - .line = S3C64XX_GPC(3), -}; - static struct wm0010_pdata wm0010_pdata = { .gpio_reset = S3C64XX_GPN(6), .reset_active_high = 1, /* Active high for Glenfarclas Rev 2 */ @@ -49,7 +45,6 @@ static struct spi_board_info wm1253_devs[] = { .chip_select = 0, .mode = SPI_MODE_0, .irq = S3C_EINT(4), - .controller_data = &wm0010_spi_csinfo, .platform_data = &wm0010_pdata, }, }; @@ -62,7 +57,6 @@ static struct spi_board_info balblair_devs[] = { .chip_select = 0, .mode = SPI_MODE_0, .irq = S3C_EINT(4), - .controller_data = &wm0010_spi_csinfo, .platform_data = &wm0010_pdata, }, }; @@ -229,10 +223,6 @@ static struct arizona_pdata wm5102_reva_pdata = { }, }; -static struct s3c64xx_spi_csinfo codec_spi_csinfo = { - .line = S3C64XX_GPN(5), -}; - static struct spi_board_info wm5102_reva_spi_devs[] = { [0] = { .modalias = "wm5102", @@ -242,7 +232,6 @@ static struct spi_board_info wm5102_reva_spi_devs[] = { .mode = SPI_MODE_0, .irq = GLENFARCLAS_PMIC_IRQ_BASE + WM831X_IRQ_GPIO_2, - .controller_data = &codec_spi_csinfo, .platform_data = &wm5102_reva_pdata, }, }; @@ -275,7 +264,6 @@ static struct spi_board_info wm5102_spi_devs[] = { .mode = SPI_MODE_0, .irq = GLENFARCLAS_PMIC_IRQ_BASE + WM831X_IRQ_GPIO_2, - .controller_data = &codec_spi_csinfo, .platform_data = &wm5102_pdata, }, }; @@ -298,7 +286,6 @@ static struct spi_board_info wm5110_spi_devs[] = { .mode = SPI_MODE_0, .irq = GLENFARCLAS_PMIC_IRQ_BASE + WM831X_IRQ_GPIO_2, - .controller_data = &codec_spi_csinfo, .platform_data = &wm5102_reva_pdata, }, }; diff --git a/arch/arm/mach-s3c/mach-crag6410.c b/arch/arm/mach-s3c/mach-crag6410.c index 4a12c75d407f..e3e0fe897bcc 100644 --- a/arch/arm/mach-s3c/mach-crag6410.c +++ b/arch/arm/mach-s3c/mach-crag6410.c @@ -825,6 +825,15 @@ static const struct gpio_led_platform_data gpio_leds_pdata = { static struct dwc2_hsotg_plat crag6410_hsotg_pdata; +static struct gpiod_lookup_table crag_spi0_gpiod_table = { + .dev_id = "s3c6410-spi.0", + .table = { + GPIO_LOOKUP_IDX("GPIOC", 3, "cs", 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("GPION", 5, "cs", 1, GPIO_ACTIVE_LOW), + { }, + }, +}; + static void __init crag6410_machine_init(void) { /* Open drain IRQs need pullups */ @@ -856,7 +865,9 @@ static void __init crag6410_machine_init(void) i2c_register_board_info(1, i2c_devs1, ARRAY_SIZE(i2c_devs1)); samsung_keypad_set_platdata(&crag6410_keypad_data); - s3c64xx_spi0_set_platdata(NULL, 0, 2); + + gpiod_add_lookup_table(&crag_spi0_gpiod_table); + s3c64xx_spi0_set_platdata(0, 2); pwm_add_table(crag6410_pwm_lookup, ARRAY_SIZE(crag6410_pwm_lookup)); platform_add_devices(crag6410_devices, ARRAY_SIZE(crag6410_devices)); diff --git a/arch/arm/mach-s3c/setup-spi-s3c64xx.c b/arch/arm/mach-s3c/setup-spi-s3c64xx.c index efcf78d41585..497aff71c29c 100644 --- a/arch/arm/mach-s3c/setup-spi-s3c64xx.c +++ b/arch/arm/mach-s3c/setup-spi-s3c64xx.c @@ -16,12 +16,3 @@ int s3c64xx_spi0_cfg_gpio(void) return 0; } #endif - -#ifdef CONFIG_S3C64XX_DEV_SPI1 -int s3c64xx_spi1_cfg_gpio(void) -{ - s3c_gpio_cfgall_range(S3C64XX_GPC(4), 3, - S3C_GPIO_SFN(2), S3C_GPIO_PULL_UP); - return 0; -} -#endif diff --git a/arch/arm/mach-s3c/spi-core-s3c24xx.h b/arch/arm/mach-s3c/spi-core-s3c24xx.h index 057667469cc3..919c5fd0c9af 100644 --- a/arch/arm/mach-s3c/spi-core-s3c24xx.h +++ b/arch/arm/mach-s3c/spi-core-s3c24xx.h @@ -16,12 +16,6 @@ static inline void s3c24xx_spi_setname(char *name) #ifdef CONFIG_S3C64XX_DEV_SPI0 s3c64xx_device_spi0.name = name; #endif -#ifdef CONFIG_S3C64XX_DEV_SPI1 - s3c64xx_device_spi1.name = name; -#endif -#ifdef CONFIG_S3C64XX_DEV_SPI2 - s3c64xx_device_spi2.name = name; -#endif } #endif /* __PLAT_S3C_SPI_CORE_S3C24XX_H */ diff --git a/arch/arm/mach-socfpga/Kconfig b/arch/arm/mach-socfpga/Kconfig index 43ddec677c0b..594edf9bbea4 100644 --- a/arch/arm/mach-socfpga/Kconfig +++ b/arch/arm/mach-socfpga/Kconfig @@ -2,6 +2,7 @@ menuconfig ARCH_INTEL_SOCFPGA bool "Altera SOCFPGA family" depends on ARCH_MULTI_V7 + select ARCH_HAS_RESET_CONTROLLER select ARCH_SUPPORTS_BIG_ENDIAN select ARM_AMBA select ARM_GIC @@ -18,6 +19,7 @@ menuconfig ARCH_INTEL_SOCFPGA select PL310_ERRATA_727915 select PL310_ERRATA_753970 if PL310 select PL310_ERRATA_769419 + select RESET_CONTROLLER if ARCH_INTEL_SOCFPGA config SOCFPGA_SUSPEND diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 58afba346729..d30ee26ccc87 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -386,6 +386,7 @@ config CPU_V6 select CPU_PABRT_V6 select CPU_THUMB_CAPABLE select CPU_TLB_V6 if MMU + select SMP_ON_UP if SMP # ARMv6k config CPU_V6K @@ -830,6 +831,7 @@ config CPU_BPREDICT_DISABLE config CPU_SPECTRE bool + select GENERIC_CPU_VULNERABILITIES config HARDEN_BRANCH_PREDICTOR bool "Harden the branch predictor against aliasing attacks" if EXPERT @@ -850,6 +852,16 @@ config HARDEN_BRANCH_PREDICTOR If unsure, say Y. +config HARDEN_BRANCH_HISTORY + bool "Harden Spectre style attacks against branch history" if EXPERT + depends on CPU_SPECTRE + default y + help + Speculation attacks against some high-performance processors can + make use of branch history to influence future speculation. When + taking an exception, a sequence of branches overwrites the branch + history, or branch history is invalidated. + config TLS_REG_EMUL bool select NEED_KUSER_HELPERS diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 830bbfb26ca5..7c9499b728c4 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -90,7 +90,7 @@ ENDPROC(v7_flush_icache_all) * * Flush the D-cache up to the Level of Unification Inner Shareable * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 */ ENTRY(v7_flush_dcache_louis) @@ -117,7 +117,7 @@ ENDPROC(v7_flush_dcache_louis) * * Flush the whole D-cache. * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 * * - mm - mm_struct describing address space */ @@ -149,22 +149,22 @@ flush_levels: movw r4, #0x3ff ands r4, r4, r1, lsr #3 @ find maximum number on the way size clz r5, r4 @ find bit position of way size increment - movw r7, #0x7fff - ands r7, r7, r1, lsr #13 @ extract max number of the index size + movw r6, #0x7fff + and r1, r6, r1, lsr #13 @ extract max number of the index size + mov r6, #1 + movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...] + movne r6, r6, lsl r5 @ 1 shifted left by same amount loop1: - mov r9, r7 @ create working copy of max index + mov r9, r1 @ create working copy of max index loop2: - ARM( orr r11, r10, r4, lsl r5 ) @ factor way and cache number into r11 - THUMB( lsl r6, r4, r5 ) - THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 - ARM( orr r11, r11, r9, lsl r2 ) @ factor index number into r11 - THUMB( lsl r6, r9, r2 ) - THUMB( orr r11, r11, r6 ) @ factor index number into r11 - mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way + mov r5, r9, lsl r2 @ factor set number into r5 + orr r5, r5, r4 @ factor way number into r5 + orr r5, r5, r10 @ factor cache level into r5 + mcr p15, 0, r5, c7, c14, 2 @ clean & invalidate by set/way subs r9, r9, #1 @ decrement the index bge loop2 - subs r4, r4, #1 @ decrement the way - bge loop1 + subs r4, r4, r6 @ decrement the way + bcs loop1 skip: add r10, r10, #2 @ increment cache number cmp r3, r10 @@ -192,14 +192,12 @@ ENDPROC(v7_flush_dcache_all) * */ ENTRY(v7_flush_kern_cache_all) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_all mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr ENDPROC(v7_flush_kern_cache_all) @@ -210,14 +208,12 @@ ENDPROC(v7_flush_kern_cache_all) * Invalidate the I-cache to the point of unification. */ ENTRY(v7_flush_kern_cache_louis) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_louis mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr ENDPROC(v7_flush_kern_cache_louis) diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 48091870db89..4204ffa2d104 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -240,8 +240,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) unsigned int cpu = smp_processor_id(); u64 asid; - if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) - __check_vmalloc_seq(mm); + check_vmalloc_seq(mm); /* * We cannot update the pgd and the ASID atomicly with classic diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 6d0cb0f7bc54..fe249ea91908 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -164,47 +164,6 @@ phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) return phys; } -static void __init arm_initrd_init(void) -{ -#ifdef CONFIG_BLK_DEV_INITRD - phys_addr_t start; - unsigned long size; - - initrd_start = initrd_end = 0; - - if (!phys_initrd_size) - return; - - /* - * Round the memory region to page boundaries as per free_initrd_mem() - * This allows us to detect whether the pages overlapping the initrd - * are in use, but more importantly, reserves the entire set of pages - * as we don't want these pages allocated for other purposes. - */ - start = round_down(phys_initrd_start, PAGE_SIZE); - size = phys_initrd_size + (phys_initrd_start - start); - size = round_up(size, PAGE_SIZE); - - if (!memblock_is_region_memory(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", - (u64)start, size); - return; - } - - if (memblock_is_region_reserved(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", - (u64)start, size); - return; - } - - memblock_reserve(start, size); - - /* Now convert initrd to virtual addresses */ - initrd_start = __phys_to_virt(phys_initrd_start); - initrd_end = initrd_start + phys_initrd_size; -#endif -} - #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND void check_cpu_icache_size(int cpuid) { @@ -226,7 +185,7 @@ void __init arm_memblock_init(const struct machine_desc *mdesc) /* Register the kernel text, kernel data and initrd with memblock. */ memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); - arm_initrd_init(); + reserve_initrd_mem(); arm_mm_memblock_reserve(); diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 197f8eb3a775..aa08bcb72db9 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -117,16 +117,21 @@ EXPORT_SYMBOL(ioremap_page); void __check_vmalloc_seq(struct mm_struct *mm) { - unsigned int seq; + int seq; do { - seq = init_mm.context.vmalloc_seq; + seq = atomic_read(&init_mm.context.vmalloc_seq); memcpy(pgd_offset(mm, VMALLOC_START), pgd_offset_k(VMALLOC_START), sizeof(pgd_t) * (pgd_index(VMALLOC_END) - pgd_index(VMALLOC_START))); - mm->context.vmalloc_seq = seq; - } while (seq != init_mm.context.vmalloc_seq); + /* + * Use a store-release so that other CPUs that observe the + * counter's new value are guaranteed to see the results of the + * memcpy as well. + */ + atomic_set_release(&mm->context.vmalloc_seq, seq); + } while (seq != atomic_read(&init_mm.context.vmalloc_seq)); } #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) @@ -157,7 +162,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Note: this is still racy on SMP machines. */ pmd_clear(pmdp); - init_mm.context.vmalloc_seq++; + atomic_inc_return_release(&init_mm.context.vmalloc_seq); /* * Free the page table, if there was one. @@ -174,8 +179,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Ensure that the active_mm is up to date - we want to * catch any use-after-iounmap cases. */ - if (current->active_mm->context.vmalloc_seq != init_mm.context.vmalloc_seq) - __check_vmalloc_seq(current->active_mm); + check_vmalloc_seq(current->active_mm); flush_tlb_kernel_range(virt, end); } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 274e4f73fd33..5e2be37a198e 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -212,12 +212,14 @@ early_param("ecc", early_ecc); static int __init early_cachepolicy(char *p) { pr_warn("cachepolicy kernel parameter not supported without cp15\n"); + return 0; } early_param("cachepolicy", early_cachepolicy); static int __init noalign_setup(char *__unused) { pr_warn("noalign kernel parameter not supported without cp15\n"); + return 1; } __setup("noalign", noalign_setup); diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index 114c05ab4dd9..06dbfb968182 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -6,8 +6,35 @@ #include <asm/cp15.h> #include <asm/cputype.h> #include <asm/proc-fns.h> +#include <asm/spectre.h> #include <asm/system_misc.h> +#ifdef CONFIG_ARM_PSCI +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_ARCH_WORKAROUND_1, &res); + + switch ((int)res.a0) { + case SMCCC_RET_SUCCESS: + return SPECTRE_MITIGATED; + + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: + return SPECTRE_UNAFFECTED; + + default: + return SPECTRE_VULNERABLE; + } +} +#else +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + return SPECTRE_VULNERABLE; +} +#endif + #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); @@ -36,13 +63,61 @@ static void __maybe_unused call_hvc_arch_workaround_1(void) arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void cpu_v7_spectre_init(void) +static unsigned int spectre_v2_install_workaround(unsigned int method) { const char *spectre_v2_method = NULL; int cpu = smp_processor_id(); if (per_cpu(harden_branch_predictor_fn, cpu)) - return; + return SPECTRE_MITIGATED; + + switch (method) { + case SPECTRE_V2_METHOD_BPIALL: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_bpiall; + spectre_v2_method = "BPIALL"; + break; + + case SPECTRE_V2_METHOD_ICIALLU: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_iciallu; + spectre_v2_method = "ICIALLU"; + break; + + case SPECTRE_V2_METHOD_HVC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_hvc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_hvc_switch_mm; + spectre_v2_method = "hypervisor"; + break; + + case SPECTRE_V2_METHOD_SMC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_smc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_smc_switch_mm; + spectre_v2_method = "firmware"; + break; + } + + if (spectre_v2_method) + pr_info("CPU%u: Spectre v2: using %s workaround\n", + smp_processor_id(), spectre_v2_method); + + return SPECTRE_MITIGATED; +} +#else +static unsigned int spectre_v2_install_workaround(unsigned int method) +{ + pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n", + smp_processor_id()); + + return SPECTRE_VULNERABLE; +} +#endif + +static void cpu_v7_spectre_v2_init(void) +{ + unsigned int state, method = 0; switch (read_cpuid_part()) { case ARM_CPU_PART_CORTEX_A8: @@ -51,69 +126,133 @@ static void cpu_v7_spectre_init(void) case ARM_CPU_PART_CORTEX_A17: case ARM_CPU_PART_CORTEX_A73: case ARM_CPU_PART_CORTEX_A75: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_bpiall; - spectre_v2_method = "BPIALL"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; break; case ARM_CPU_PART_CORTEX_A15: case ARM_CPU_PART_BRAHMA_B15: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_iciallu; - spectre_v2_method = "ICIALLU"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_ICIALLU; break; -#ifdef CONFIG_ARM_PSCI case ARM_CPU_PART_BRAHMA_B53: /* Requires no workaround */ + state = SPECTRE_UNAFFECTED; break; + default: /* Other ARM CPUs require no workaround */ - if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) + if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { + state = SPECTRE_UNAFFECTED; break; + } + fallthrough; - /* Cortex A57/A72 require firmware workaround */ - case ARM_CPU_PART_CORTEX_A57: - case ARM_CPU_PART_CORTEX_A72: { - struct arm_smccc_res res; - arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, - ARM_SMCCC_ARCH_WORKAROUND_1, &res); - if ((int)res.a0 != 0) - return; + /* Cortex A57/A72 require firmware workaround */ + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = spectre_v2_get_cpu_fw_mitigation_state(); + if (state != SPECTRE_MITIGATED) + break; switch (arm_smccc_1_1_get_conduit()) { case SMCCC_CONDUIT_HVC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_hvc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_hvc_switch_mm; - spectre_v2_method = "hypervisor"; + method = SPECTRE_V2_METHOD_HVC; break; case SMCCC_CONDUIT_SMC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_smc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_smc_switch_mm; - spectre_v2_method = "firmware"; + method = SPECTRE_V2_METHOD_SMC; break; default: + state = SPECTRE_VULNERABLE; break; } } -#endif + + if (state == SPECTRE_MITIGATED) + state = spectre_v2_install_workaround(method); + + spectre_v2_update_state(state, method); +} + +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +static int spectre_bhb_method; + +static const char *spectre_bhb_method_name(int method) +{ + switch (method) { + case SPECTRE_V2_METHOD_LOOP8: + return "loop"; + + case SPECTRE_V2_METHOD_BPIALL: + return "BPIALL"; + + default: + return "unknown"; } +} - if (spectre_v2_method) - pr_info("CPU%u: Spectre v2: using %s workaround\n", - smp_processor_id(), spectre_v2_method); +static int spectre_bhb_install_workaround(int method) +{ + if (spectre_bhb_method != method) { + if (spectre_bhb_method) { + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", + smp_processor_id()); + + return SPECTRE_VULNERABLE; + } + + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) + return SPECTRE_VULNERABLE; + + spectre_bhb_method = method; + } + + pr_info("CPU%u: Spectre BHB: using %s workaround\n", + smp_processor_id(), spectre_bhb_method_name(method)); + + return SPECTRE_MITIGATED; } #else -static void cpu_v7_spectre_init(void) +static int spectre_bhb_install_workaround(int method) { + return SPECTRE_VULNERABLE; } #endif +static void cpu_v7_spectre_bhb_init(void) +{ + unsigned int state, method = 0; + + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_BRAHMA_B15: + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_LOOP8; + break; + + case ARM_CPU_PART_CORTEX_A73: + case ARM_CPU_PART_CORTEX_A75: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; + break; + + default: + state = SPECTRE_UNAFFECTED; + break; + } + + if (state == SPECTRE_MITIGATED) + state = spectre_bhb_install_workaround(method); + + spectre_v2_update_state(state, method); +} + static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, u32 mask, const char *msg) { @@ -142,16 +281,17 @@ static bool check_spectre_auxcr(bool *warned, u32 bit) void cpu_v7_ca8_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } void cpu_v7_ca15_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } void cpu_v7_bugs_init(void) { - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); + cpu_v7_spectre_bhb_init(); } diff --git a/arch/arm/probes/kprobes/actions-common.c b/arch/arm/probes/kprobes/actions-common.c index 836aebe596cd..79171344dbeb 100644 --- a/arch/arm/probes/kprobes/actions-common.c +++ b/arch/arm/probes/kprobes/actions-common.c @@ -84,7 +84,8 @@ emulate_generic_r0_12_noflags(probes_opcode_t insn, register void *rfn asm("lr") = asi->insn_fn; __asm__ __volatile__ ( - "stmdb sp!, {%[regs], r11} \n\t" +ARM( "stmdb sp!, {%[regs], r11} \n\t" ) +THUMB( "stmdb sp!, {%[regs], r7} \n\t" ) "ldmia %[regs], {r0-r12} \n\t" #if __LINUX_ARM_ARCH__ >= 6 "blx %[fn] \n\t" @@ -96,10 +97,11 @@ emulate_generic_r0_12_noflags(probes_opcode_t insn, #endif "ldr lr, [sp], #4 \n\t" /* lr = regs */ "stmia lr, {r0-r12} \n\t" - "ldr r11, [sp], #4 \n\t" +ARM( "ldr r11, [sp], #4 \n\t" ) +THUMB( "ldr r7, [sp], #4 \n\t" ) : [regs] "=r" (rregs), [fn] "=r" (rfn) : "0" (rregs), "1" (rfn) - : "r0", "r2", "r3", "r4", "r5", "r6", "r7", + : "r0", "r2", "r3", "r4", "r5", "r6", ARM("r7") THUMB("r11"), "r8", "r9", "r10", "r12", "memory", "cc" ); } diff --git a/arch/arm/probes/kprobes/actions-thumb.c b/arch/arm/probes/kprobes/actions-thumb.c index 7884fcb81c26..51624fc263fc 100644 --- a/arch/arm/probes/kprobes/actions-thumb.c +++ b/arch/arm/probes/kprobes/actions-thumb.c @@ -447,14 +447,16 @@ t16_emulate_loregs(probes_opcode_t insn, __asm__ __volatile__ ( "msr cpsr_fs, %[oldcpsr] \n\t" + "mov r11, r7 \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" + "mov r7, r11 \n\t" "mrs %[newcpsr], cpsr \n\t" : [newcpsr] "=r" (newcpsr) : [oldcpsr] "r" (oldcpsr), [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r11", "lr", "memory", "cc" ); @@ -524,14 +526,16 @@ t16_emulate_push(probes_opcode_t insn, struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldr r8, [%[regs], #14*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r8", "r9", "r11", "lr", "memory", "cc" ); } @@ -558,14 +562,16 @@ t16_emulate_pop_nopc(probes_opcode_t insn, struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r9", "r11", "lr", "memory", "cc" ); } @@ -577,14 +583,16 @@ t16_emulate_pop_pc(probes_opcode_t insn, register unsigned long pc asm("r8"); __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : "=r" (pc) : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r9", "r11", "lr", "memory", "cc" ); diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile index 7c9e395b77f7..ec52b776f926 100644 --- a/arch/arm/vdso/Makefile +++ b/arch/arm/vdso/Makefile @@ -18,7 +18,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO32 ldflags-$(CONFIG_CPU_ENDIAN_BE8) := --be8 ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - -z max-page-size=4096 -nostdlib -shared $(ldflags-y) \ + -z max-page-size=4096 -shared $(ldflags-y) \ --hash-style=sysv --build-id=sha1 \ -T diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index cbcd42decb2a..13aa8d955c46 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -10,6 +10,7 @@ config ARM64 select ACPI_SPCR_TABLE if ACPI select ACPI_PPTT if ACPI select ARCH_HAS_DEBUG_WX + select ARCH_BINFMT_ELF_EXTRA_PHDRS select ARCH_BINFMT_ELF_STATE select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION @@ -18,6 +19,7 @@ config ARM64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DMA_PREP_COHERENT @@ -192,6 +194,7 @@ config ARM64 select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_FUNCTION_ARG_ACCESS_API @@ -672,6 +675,7 @@ config ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE config ARM64_ERRATUM_2051678 bool "Cortex-A510: 2051678: disable Hardware Update of the page table dirty bit" + default y help This options adds the workaround for ARM Cortex-A510 erratum ARM64_ERRATUM_2051678. Affected Coretex-A510 might not respect the ordering rules for @@ -890,13 +894,17 @@ config CAVIUM_ERRATUM_23144 If unsure, say Y. config CAVIUM_ERRATUM_23154 - bool "Cavium erratum 23154: Access to ICC_IAR1_EL1 is not sync'ed" + bool "Cavium errata 23154 and 38545: GICv3 lacks HW synchronisation" default y help - The gicv3 of ThunderX requires a modified version for + The ThunderX GICv3 implementation requires a modified version for reading the IAR status to ensure data synchronization (access to icc_iar1_el1 is not sync'ed before and after). + It also suffers from erratum 38545 (also present on Marvell's + OcteonTX and OcteonTX2), resulting in deactivated interrupts being + spuriously presented to the CPU interface. + If unsure, say Y. config CAVIUM_ERRATUM_27456 @@ -1251,10 +1259,7 @@ config HW_PERF_EVENTS def_bool y depends on ARM_PMU -config ARCH_HAS_FILTER_PGPROT - def_bool y - -# Supported by clang >= 7.0 +# Supported by clang >= 7.0 or GCC >= 12.0.0 config CC_HAVE_SHADOW_CALL_STACK def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) @@ -1382,6 +1387,15 @@ config UNMAP_KERNEL_AT_EL0 If unsure, say Y. +config MITIGATE_SPECTRE_BRANCH_HISTORY + bool "Mitigate Spectre style attacks against branch history" if EXPERT + default y + help + Speculation attacks against some high-performance processors can + make use of branch history to influence future speculation. + When taking an exception from user-space, a sequence of branches + or a firmware call overwrites the branch history. + config RODATA_FULL_DEFAULT_ENABLED bool "Apply r/o permissions of VM areas also to their linear aliases" default y diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index 7d5d58800170..21697449d762 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -309,9 +309,6 @@ config ARCH_VISCONTI help This enables support for Toshiba Visconti SoCs Family. -config ARCH_VULCAN - def_bool n - config ARCH_XGENE bool "AppliedMicro X-Gene SOC Family" help diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index 517519e6e87f..f84d4b489e0b 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -107,6 +107,12 @@ no-map; }; + /* 32 MiB reserved for ARM Trusted Firmware (BL32) */ + secmon_reserved_bl32: secmon@5300000 { + reg = <0x0 0x05300000 0x0 0x2000000>; + no-map; + }; + linux,cma { compatible = "shared-dma-pool"; reusable; diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts index d8838dde0f0f..4fb31c2ba31c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts +++ b/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts @@ -157,14 +157,6 @@ regulator-always-on; }; - reserved-memory { - /* TEE Reserved Memory */ - bl32_reserved: bl32@5000000 { - reg = <0x0 0x05300000 0x0 0x2000000>; - no-map; - }; - }; - sdio_pwrseq: sdio-pwrseq { compatible = "mmc-pwrseq-simple"; reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>; diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi index 3e968b244191..fd3fa82e4c33 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi @@ -17,7 +17,7 @@ rtc1 = &vrtc; }; - dioo2133: audio-amplifier-0 { + dio2133: audio-amplifier-0 { compatible = "simple-audio-amplifier"; enable-gpios = <&gpio_ao GPIOAO_2 GPIO_ACTIVE_HIGH>; VCC-supply = <&vcc_5v>; @@ -219,7 +219,7 @@ audio-widgets = "Line", "Lineout"; audio-aux-devs = <&tdmout_b>, <&tdmout_c>, <&tdmin_a>, <&tdmin_b>, <&tdmin_c>, <&tdmin_lb>, - <&dioo2133>; + <&dio2133>; audio-routing = "TDMOUT_B IN 0", "FRDDR_A OUT 1", "TDMOUT_B IN 1", "FRDDR_B OUT 1", "TDMOUT_B IN 2", "FRDDR_C OUT 1", diff --git a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi index 6b457b2c30a4..aa14ea017a61 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi @@ -49,6 +49,12 @@ no-map; }; + /* 32 MiB reserved for ARM Trusted Firmware (BL32) */ + secmon_reserved_bl32: secmon@5300000 { + reg = <0x0 0x05300000 0x0 0x2000000>; + no-map; + }; + linux,cma { compatible = "shared-dma-pool"; reusable; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts index 212c6aa5a3b8..5751c48620ed 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts @@ -123,7 +123,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; - enable-gpio = <&gpio GPIOE_2 GPIO_ACTIVE_HIGH>; + enable-gpio = <&gpio_ao GPIOE_2 GPIO_ACTIVE_HIGH>; enable-active-high; regulator-always-on; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi index 0bd1e98a0eef..ddb1b345397f 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi @@ -48,7 +48,7 @@ regulator-max-microvolt = <3300000>; vin-supply = <&vcc_5v>; - enable-gpio = <&gpio GPIOE_2 GPIO_ACTIVE_HIGH>; + enable-gpio = <&gpio_ao GPIOE_2 GPIO_OPEN_DRAIN>; enable-active-high; regulator-always-on; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts index 427475846fc7..a5d79f2f7c19 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts @@ -203,14 +203,6 @@ regulator-always-on; }; - reserved-memory { - /* TEE Reserved Memory */ - bl32_reserved: bl32@5000000 { - reg = <0x0 0x05300000 0x0 0x2000000>; - no-map; - }; - }; - sdio_pwrseq: sdio-pwrseq { compatible = "mmc-pwrseq-simple"; reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 19afbc91020a..9f8f4145db88 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -97,6 +97,18 @@ <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>; }; + pmu-e { + compatible = "apple,icestorm-pmu"; + interrupt-parent = <&aic>; + interrupts = <AIC_FIQ AIC_CPU_PMU_E IRQ_TYPE_LEVEL_HIGH>; + }; + + pmu-p { + compatible = "apple,firestorm-pmu"; + interrupt-parent = <&aic>; + interrupts = <AIC_FIQ AIC_CPU_PMU_P IRQ_TYPE_LEVEL_HIGH>; + }; + clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; @@ -213,6 +225,18 @@ interrupt-controller; reg = <0x2 0x3b100000 0x0 0x8000>; power-domains = <&ps_aic>; + + affinities { + e-core-pmu-affinity { + apple,fiq-index = <AIC_CPU_PMU_E>; + cpus = <&cpu0 &cpu1 &cpu2 &cpu3>; + }; + + p-core-pmu-affinity { + apple,fiq-index = <AIC_CPU_PMU_P>; + cpus = <&cpu4 &cpu5 &cpu6 &cpu7>; + }; + }; }; pmgr: power-management@23b700000 { diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi index 6288e104a089..a2635b14da30 100644 --- a/arch/arm64/boot/dts/arm/juno-base.dtsi +++ b/arch/arm64/boot/dts/arm/juno-base.dtsi @@ -543,8 +543,7 @@ <0x02000000 0x00 0x50000000 0x00 0x50000000 0x0 0x08000000>, <0x42000000 0x40 0x00000000 0x40 0x00000000 0x1 0x00000000>; /* Standard AXI Translation entries as programmed by EDK2 */ - dma-ranges = <0x02000000 0x0 0x2c1c0000 0x0 0x2c1c0000 0x0 0x00040000>, - <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x80000000>, + dma-ranges = <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x80000000>, <0x43000000 0x8 0x00000000 0x8 0x00000000 0x2 0x00000000>; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 7>; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts index d74e738e4070..c03f4e183389 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts @@ -157,6 +157,10 @@ }; }; +&ftm_alarm0 { + status = "okay"; +}; + &gpio1 { gpio-line-names = "", "", "", "", "", "", "", "", diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi index 3ed1f2c51cad..18e529118476 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi @@ -253,18 +253,18 @@ interrupt-controller; reg = <0x14 4>; interrupt-map = - <0 0 &gic 0 0 GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, - <1 0 &gic 0 0 GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, - <2 0 &gic 0 0 GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, - <3 0 &gic 0 0 GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, - <4 0 &gic 0 0 GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, - <5 0 &gic 0 0 GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, - <6 0 &gic 0 0 GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, - <7 0 &gic 0 0 GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, - <8 0 &gic 0 0 GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, - <9 0 &gic 0 0 GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, - <10 0 &gic 0 0 GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, - <11 0 &gic 0 0 GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; + <0 0 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, + <1 0 &gic GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, + <2 0 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, + <3 0 &gic GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, + <4 0 &gic GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, + <5 0 &gic GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, + <6 0 &gic GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, + <7 0 &gic GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, + <8 0 &gic GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, + <9 0 &gic GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, + <10 0 &gic GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, + <11 0 &gic GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; interrupt-map-mask = <0xffffffff 0x0>; }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi index 3cb9c21d2775..1282b61da8a5 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi @@ -293,18 +293,18 @@ interrupt-controller; reg = <0x14 4>; interrupt-map = - <0 0 &gic 0 0 GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, - <1 0 &gic 0 0 GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, - <2 0 &gic 0 0 GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, - <3 0 &gic 0 0 GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, - <4 0 &gic 0 0 GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, - <5 0 &gic 0 0 GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, - <6 0 &gic 0 0 GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, - <7 0 &gic 0 0 GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, - <8 0 &gic 0 0 GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, - <9 0 &gic 0 0 GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, - <10 0 &gic 0 0 GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, - <11 0 &gic 0 0 GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; + <0 0 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, + <1 0 &gic GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, + <2 0 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, + <3 0 &gic GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, + <4 0 &gic GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, + <5 0 &gic GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, + <6 0 &gic GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, + <7 0 &gic GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, + <8 0 &gic GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, + <9 0 &gic GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, + <10 0 &gic GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, + <11 0 &gic GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; interrupt-map-mask = <0xffffffff 0x0>; }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 7032505f5ef3..3c611cb4f5fe 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -680,18 +680,18 @@ interrupt-controller; reg = <0x14 4>; interrupt-map = - <0 0 &gic 0 0 GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, - <1 0 &gic 0 0 GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, - <2 0 &gic 0 0 GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, - <3 0 &gic 0 0 GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, - <4 0 &gic 0 0 GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, - <5 0 &gic 0 0 GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, - <6 0 &gic 0 0 GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, - <7 0 &gic 0 0 GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, - <8 0 &gic 0 0 GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, - <9 0 &gic 0 0 GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, - <10 0 &gic 0 0 GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, - <11 0 &gic 0 0 GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; + <0 0 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, + <1 0 &gic GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, + <2 0 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, + <3 0 &gic GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, + <4 0 &gic GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, + <5 0 &gic GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, + <6 0 &gic GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, + <7 0 &gic GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, + <8 0 &gic GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, + <9 0 &gic GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, + <10 0 &gic GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, + <11 0 &gic GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; interrupt-map-mask = <0xffffffff 0x0>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index f77f90ed416f..0c7a72c51a31 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -707,7 +707,6 @@ clocks = <&clk IMX8MM_CLK_VPU_DEC_ROOT>; assigned-clocks = <&clk IMX8MM_CLK_VPU_BUS>; assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_800M>; - resets = <&src IMX8MQ_RESET_VPU_RESET>; }; pgc_vpu_g1: power-domain@7 { diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi index f3e3418f7edc..2d4a472af6a9 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi @@ -1115,8 +1115,8 @@ status = "okay"; ports { - port@1 { - reg = <1>; + port@0 { + reg = <0>; mipi1_sensor_ep: endpoint { remote-endpoint = <&camera1_ep>; diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index 2df2510d0118..e92ebb6147e6 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -554,7 +554,7 @@ assigned-clock-rates = <0>, <0>, <0>, <594000000>; status = "disabled"; - port@0 { + port { lcdif_mipi_dsi: endpoint { remote-endpoint = <&mipi_dsi_lcdif_in>; }; @@ -1151,8 +1151,8 @@ #address-cells = <1>; #size-cells = <0>; - port@0 { - reg = <0>; + port@1 { + reg = <1>; csi1_mipi_ep: endpoint { remote-endpoint = <&csi1_ep>; @@ -1203,8 +1203,8 @@ #address-cells = <1>; #size-cells = <0>; - port@0 { - reg = <0>; + port@1 { + reg = <1>; csi2_mipi_ep: endpoint { remote-endpoint = <&csi2_ep>; diff --git a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi index a987ff7156bd..09f7364dd1d0 100644 --- a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi @@ -132,7 +132,7 @@ scmi_sensor: protocol@15 { reg = <0x15>; - #thermal-sensor-cells = <0>; + #thermal-sensor-cells = <1>; }; }; }; diff --git a/arch/arm64/boot/dts/freescale/mba8mx.dtsi b/arch/arm64/boot/dts/freescale/mba8mx.dtsi index f27e3c8de916..ce6d5bdba0a8 100644 --- a/arch/arm64/boot/dts/freescale/mba8mx.dtsi +++ b/arch/arm64/boot/dts/freescale/mba8mx.dtsi @@ -91,7 +91,7 @@ sound { compatible = "fsl,imx-audio-tlv320aic32x4"; - model = "tqm-tlv320aic32"; + model = "imx-audio-tlv320aic32x4"; ssi-controller = <&sai3>; audio-codec = <&tlv320aic3x04>; }; diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi index 0dd2d2ee765a..f4270cf18996 100644 --- a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi +++ b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi @@ -502,7 +502,7 @@ }; usb0: usb@ffb00000 { - compatible = "snps,dwc2"; + compatible = "intel,socfpga-agilex-hsotg", "snps,dwc2"; reg = <0xffb00000 0x40000>; interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>; phys = <&usbphy0>; @@ -515,7 +515,7 @@ }; usb1: usb@ffb40000 { - compatible = "snps,dwc2"; + compatible = "intel,socfpga-agilex-hsotg", "snps,dwc2"; reg = <0xffb40000 0x40000>; interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>; phys = <&usbphy0>; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index 04da07ae4420..1cee26479bfe 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -18,6 +18,7 @@ aliases { spi0 = &spi0; + ethernet0 = ð0; ethernet1 = ð1; mmc0 = &sdhci0; mmc1 = &sdhci1; @@ -138,7 +139,9 @@ /* * U-Boot port for Turris Mox has a bug which always expects that "ranges" DT property * contains exactly 2 ranges with 3 (child) address cells, 2 (parent) address cells and - * 2 size cells and also expects that the second range starts at 16 MB offset. If these + * 2 size cells and also expects that the second range starts at 16 MB offset. Also it + * expects that first range uses same address for PCI (child) and CPU (parent) cells (so + * no remapping) and that this address is the lowest from all specified ranges. If these * conditions are not met then U-Boot crashes during loading kernel DTB file. PCIe address * space is 128 MB long, so the best split between MEM and IO is to use fixed 16 MB window * for IO and the rest 112 MB (64+32+16) for MEM, despite that maximal IO size is just 64 kB. @@ -147,6 +150,9 @@ * https://source.denx.de/u-boot/u-boot/-/commit/cb2ddb291ee6fcbddd6d8f4ff49089dfe580f5d7 * https://source.denx.de/u-boot/u-boot/-/commit/c64ac3b3185aeb3846297ad7391fc6df8ecd73bf * https://source.denx.de/u-boot/u-boot/-/commit/4a82fca8e330157081fc132a591ebd99ba02ee33 + * Bug related to requirement of same child and parent addresses for first range is fixed + * in U-Boot version 2022.04 by following commit: + * https://source.denx.de/u-boot/u-boot/-/commit/1fd54253bca7d43d046bba4853fe5fafd034bc17 */ #address-cells = <3>; #size-cells = <2>; diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi index 673f4906eef9..fb78ef613b29 100644 --- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi @@ -499,7 +499,7 @@ * (totaling 127 MiB) for MEM. */ ranges = <0x82000000 0 0xe8000000 0 0xe8000000 0 0x07f00000 /* Port 0 MEM */ - 0x81000000 0 0xefff0000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */ + 0x81000000 0 0x00000000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */ interrupt-map-mask = <0 0 0 7>; interrupt-map = <0 0 0 1 &pcie_intc 0>, <0 0 0 2 &pcie_intc 1>, diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index 2d48c3715fc6..aaa00da5351d 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -1584,7 +1584,7 @@ #iommu-cells = <1>; nvidia,memory-controller = <&mc>; - status = "okay"; + status = "disabled"; }; smmu: iommu@12000000 { diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index 58845a14805f..e2b9ec134cb1 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -807,3 +807,8 @@ qcom,snoc-host-cap-8bit-quirk; }; + +&crypto { + /* FIXME: qce_start triggers an SError */ + status= "disable"; +}; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index 53b39e718fb6..4b19744bcfb3 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -35,6 +35,24 @@ clock-frequency = <32000>; #clock-cells = <0>; }; + + ufs_phy_rx_symbol_0_clk: ufs-phy-rx-symbol-0 { + compatible = "fixed-clock"; + clock-frequency = <1000>; + #clock-cells = <0>; + }; + + ufs_phy_rx_symbol_1_clk: ufs-phy-rx-symbol-1 { + compatible = "fixed-clock"; + clock-frequency = <1000>; + #clock-cells = <0>; + }; + + ufs_phy_tx_symbol_0_clk: ufs-phy-tx-symbol-0 { + compatible = "fixed-clock"; + clock-frequency = <1000>; + #clock-cells = <0>; + }; }; cpus { @@ -603,9 +621,9 @@ <0>, <0>, <0>, - <0>, - <0>, - <0>, + <&ufs_phy_rx_symbol_0_clk>, + <&ufs_phy_rx_symbol_1_clk>, + <&ufs_phy_tx_symbol_0_clk>, <0>, <0>; }; @@ -1923,8 +1941,8 @@ <75000000 300000000>, <0 0>, <0 0>, - <75000000 300000000>, - <75000000 300000000>; + <0 0>, + <0 0>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 10c25ad2d0c7..02b97e838c47 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -726,7 +726,7 @@ compatible = "qcom,sm8450-smmu-500", "arm,mmu-500"; reg = <0 0x15000000 0 0x100000>; #iommu-cells = <2>; - #global-interrupts = <2>; + #global-interrupts = <1>; interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, @@ -813,6 +813,7 @@ <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 690 IRQ_TYPE_LEVEL_HIGH>, @@ -1072,9 +1073,10 @@ <&gcc GCC_USB30_PRIM_MASTER_CLK>, <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, - <&gcc GCC_USB30_PRIM_SLEEP_CLK>; + <&gcc GCC_USB30_PRIM_SLEEP_CLK>, + <&gcc GCC_USB3_0_CLKREF_EN>; clock-names = "cfg_noc", "core", "iface", "mock_utmi", - "sleep"; + "sleep", "xo"; assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, <&gcc GCC_USB30_PRIM_MASTER_CLK>; diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi index f972704dfe7a..56dfbb2e2fa6 100644 --- a/arch/arm64/boot/dts/rockchip/px30.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30.dtsi @@ -711,7 +711,7 @@ clock-names = "pclk", "timer"; }; - dmac: dmac@ff240000 { + dmac: dma-controller@ff240000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xff240000 0x0 0x4000>; interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi index 39db0b85b4da..b822533dc7f1 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi @@ -489,7 +489,7 @@ status = "disabled"; }; - dmac: dmac@ff1f0000 { + dmac: dma-controller@ff1f0000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xff1f0000 0x0 0x4000>; interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi index 45a5ae5d2027..162f08bca0d4 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi @@ -286,7 +286,7 @@ sound: sound { compatible = "rockchip,rk3399-gru-sound"; - rockchip,cpu = <&i2s0 &i2s2>; + rockchip,cpu = <&i2s0 &spdif>; }; }; @@ -437,10 +437,6 @@ ap_i2c_audio: &i2c8 { status = "okay"; }; -&i2s2 { - status = "okay"; -}; - &io_domains { status = "okay"; @@ -537,6 +533,17 @@ ap_i2c_audio: &i2c8 { vqmmc-supply = <&ppvar_sd_card_io>; }; +&spdif { + status = "okay"; + + /* + * SPDIF is routed internally to DP; we either don't use these pins, or + * mux them to something else. + */ + /delete-property/ pinctrl-0; + /delete-property/ pinctrl-names; +}; + &spi1 { status = "okay"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts index 292bb7e80cf3..3ae5d727e367 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts @@ -232,6 +232,7 @@ &usbdrd_dwc3_0 { dr_mode = "otg"; + extcon = <&extcon_usb3>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi index fb67db4619ea..08fa00364b42 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi @@ -25,6 +25,13 @@ }; }; + extcon_usb3: extcon-usb3 { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb3_id>; + }; + clkin_gmac: external-gmac-clock { compatible = "fixed-clock"; clock-frequency = <125000000>; @@ -422,9 +429,22 @@ <4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; }; }; + + usb3 { + usb3_id: usb3-id { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; }; &sdhci { + /* + * Signal integrity isn't great at 200MHz but 100MHz has proven stable + * enough. + */ + max-frequency = <100000000>; + bus-width = <8>; mmc-hs400-1_8v; mmc-hs400-enhanced-strobe; diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index d3cdf6f42a30..080457a68e3c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -1881,10 +1881,10 @@ interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH 0>; clocks = <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_SFR>, - <&cru PLL_VPLL>, + <&cru SCLK_HDMI_CEC>, <&cru PCLK_VIO_GRF>, - <&cru SCLK_HDMI_CEC>; - clock-names = "iahb", "isfr", "vpll", "grf", "cec"; + <&cru PLL_VPLL>; + clock-names = "iahb", "isfr", "cec", "grf", "vpll"; power-domains = <&power RK3399_PD_HDCP>; reg-io-width = <4>; rockchip,grf = <&grf>; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts index 166399b7f13f..d9eb92d59099 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts @@ -285,8 +285,6 @@ vcc_ddr: DCDC_REG3 { regulator-always-on; regulator-boot-on; - regulator-min-microvolt = <1100000>; - regulator-max-microvolt = <1100000>; regulator-initial-mode = <0x2>; regulator-name = "vcc_ddr"; regulator-state-mem { diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index 2fd313a295f8..d91df1cde736 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -32,13 +32,11 @@ clocks = <&cru SCLK_GMAC0>, <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_MAC0_REFOUT>, <&cru ACLK_GMAC0>, <&cru PCLK_GMAC0>, - <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_GMAC0_PTP_REF>, - <&cru PCLK_XPCS>; + <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_GMAC0_PTP_REF>; clock-names = "stmmaceth", "mac_clk_rx", "mac_clk_tx", "clk_mac_refout", "aclk_mac", "pclk_mac", - "clk_mac_speed", "ptp_ref", - "pclk_xpcs"; + "clk_mac_speed", "ptp_ref"; resets = <&cru SRST_A_GMAC0>; reset-names = "stmmaceth"; rockchip,grf = <&grf>; diff --git a/arch/arm64/boot/dts/rockchip/rk356x.dtsi b/arch/arm64/boot/dts/rockchip/rk356x.dtsi index a68033a23975..8ccce54ee8e7 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x.dtsi @@ -651,7 +651,7 @@ status = "disabled"; }; - dmac0: dmac@fe530000 { + dmac0: dma-controller@fe530000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xfe530000 0x0 0x4000>; interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>, @@ -662,7 +662,7 @@ #dma-cells = <1>; }; - dmac1: dmac@fe550000 { + dmac1: dma-controller@fe550000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xfe550000 0x0 0x4000>; interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts index a5a24f9f46c5..b210cc07c539 100644 --- a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts +++ b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts @@ -15,8 +15,18 @@ model = "Texas Instruments J721S2 EVM"; chosen { - stdout-path = "serial10:115200n8"; - bootargs = "console=ttyS10,115200n8 earlycon=ns16550a,mmio32,2880000"; + stdout-path = "serial2:115200n8"; + bootargs = "console=ttyS2,115200n8 earlycon=ns16550a,mmio32,2880000"; + }; + + aliases { + serial1 = &mcu_uart0; + serial2 = &main_uart8; + mmc0 = &main_sdhci0; + mmc1 = &main_sdhci1; + can0 = &main_mcan16; + can1 = &mcu_mcan0; + can2 = &mcu_mcan1; }; evm_12v0: fixedregulator-evm12v0 { diff --git a/arch/arm64/boot/dts/ti/k3-j721s2.dtsi b/arch/arm64/boot/dts/ti/k3-j721s2.dtsi index 80d3cae03e88..fe5234c40f6c 100644 --- a/arch/arm64/boot/dts/ti/k3-j721s2.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721s2.dtsi @@ -21,28 +21,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - serial0 = &wkup_uart0; - serial1 = &mcu_uart0; - serial2 = &main_uart0; - serial3 = &main_uart1; - serial4 = &main_uart2; - serial5 = &main_uart3; - serial6 = &main_uart4; - serial7 = &main_uart5; - serial8 = &main_uart6; - serial9 = &main_uart7; - serial10 = &main_uart8; - serial11 = &main_uart9; - mmc0 = &main_sdhci0; - mmc1 = &main_sdhci1; - can0 = &main_mcan16; - can1 = &mcu_mcan0; - can2 = &mcu_mcan1; - can3 = &main_mcan3; - can4 = &main_mcan5; - }; - chosen { }; cpus { diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index addfa413650b..2a965aa0188d 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -45,7 +45,7 @@ config CRYPTO_SM3_ARM64_CE tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 config CRYPTO_SM4_ARM64_CE tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)" diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 30b7cc6a7079..561dd2332571 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -24,7 +24,6 @@ #ifdef USE_V8_CRYPTO_EXTENSIONS #define MODE "ce" #define PRIO 300 -#define STRIDE 5 #define aes_expandkey ce_aes_expandkey #define aes_ecb_encrypt ce_aes_ecb_encrypt #define aes_ecb_decrypt ce_aes_ecb_decrypt @@ -42,7 +41,6 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #else #define MODE "neon" #define PRIO 200 -#define STRIDE 4 #define aes_ecb_encrypt neon_aes_ecb_encrypt #define aes_ecb_decrypt neon_aes_ecb_decrypt #define aes_cbc_encrypt neon_aes_cbc_encrypt @@ -89,7 +87,7 @@ asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int bytes, u8 const iv[]); asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[], u8 finalbuf[]); + int rounds, int bytes, u8 ctr[]); asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int bytes, u32 const rk2[], u8 iv[], @@ -458,26 +456,21 @@ static int __maybe_unused ctr_encrypt(struct skcipher_request *req) unsigned int nbytes = walk.nbytes; u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; - unsigned int tail; if (unlikely(nbytes < AES_BLOCK_SIZE)) - src = memcpy(buf, src, nbytes); + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); else if (nbytes < walk.total) nbytes &= ~(AES_BLOCK_SIZE - 1); kernel_neon_begin(); aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, - walk.iv, buf); + walk.iv); kernel_neon_end(); - tail = nbytes % (STRIDE * AES_BLOCK_SIZE); - if (tail > 0 && tail < AES_BLOCK_SIZE) - /* - * The final partial block could not be returned using - * an overlapping store, so it was passed via buf[] - * instead. - */ - memcpy(dst + nbytes - tail, buf, tail); + if (unlikely(nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } @@ -983,6 +976,7 @@ module_cpu_feature_match(AES, aes_init); module_init(aes_init); EXPORT_SYMBOL(neon_aes_ecb_encrypt); EXPORT_SYMBOL(neon_aes_cbc_encrypt); +EXPORT_SYMBOL(neon_aes_ctr_encrypt); EXPORT_SYMBOL(neon_aes_xts_encrypt); EXPORT_SYMBOL(neon_aes_xts_decrypt); #endif diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index ff01f0167ba2..dc35eb0245c5 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -321,7 +321,7 @@ AES_FUNC_END(aes_cbc_cts_decrypt) /* * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, - * int bytes, u8 ctr[], u8 finalbuf[]) + * int bytes, u8 ctr[]) */ AES_FUNC_START(aes_ctr_encrypt) @@ -414,8 +414,8 @@ ST5( st1 {v4.16b}, [x0], #16 ) .Lctrtail: /* XOR up to MAX_STRIDE * 16 - 1 bytes of in/output with v0 ... v3/v4 */ mov x16, #16 - ands x13, x4, #0xf - csel x13, x13, x16, ne + ands x6, x4, #0xf + csel x13, x6, x16, ne ST5( cmp w4, #64 - (MAX_STRIDE << 4) ) ST5( csel x14, x16, xzr, gt ) @@ -424,10 +424,10 @@ ST5( csel x14, x16, xzr, gt ) cmp w4, #32 - (MAX_STRIDE << 4) csel x16, x16, xzr, gt cmp w4, #16 - (MAX_STRIDE << 4) - ble .Lctrtail1x adr_l x12, .Lcts_permute_table add x12, x12, x13 + ble .Lctrtail1x ST5( ld1 {v5.16b}, [x1], x14 ) ld1 {v6.16b}, [x1], x15 @@ -462,11 +462,19 @@ ST5( st1 {v5.16b}, [x0], x14 ) b .Lctrout .Lctrtail1x: - csel x0, x0, x6, eq // use finalbuf if less than a full block + sub x7, x6, #16 + csel x6, x6, x7, eq + add x1, x1, x6 + add x0, x0, x6 ld1 {v5.16b}, [x1] + ld1 {v6.16b}, [x0] ST5( mov v3.16b, v4.16b ) encrypt_block v3, w3, x2, x8, w7 + ld1 {v10.16b-v11.16b}, [x12] + tbl v3.16b, {v3.16b}, v10.16b + sshr v11.16b, v11.16b, #7 eor v5.16b, v5.16b, v3.16b + bif v5.16b, v6.16b, v11.16b st1 {v5.16b}, [x0] b .Lctrout AES_FUNC_END(aes_ctr_encrypt) diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S index a3405b8c344b..d427f4556b6e 100644 --- a/arch/arm64/crypto/aes-neonbs-core.S +++ b/arch/arm64/crypto/aes-neonbs-core.S @@ -735,119 +735,67 @@ SYM_FUNC_END(aesbs_cbc_decrypt) * int blocks, u8 iv[]) */ SYM_FUNC_START_LOCAL(__xts_crypt8) - mov x6, #1 - lsl x6, x6, x23 - subs w23, w23, #8 - csel x23, x23, xzr, pl - csel x6, x6, xzr, mi + movi v18.2s, #0x1 + movi v19.2s, #0x87 + uzp1 v18.4s, v18.4s, v19.4s + + ld1 {v0.16b-v3.16b}, [x1], #64 + ld1 {v4.16b-v7.16b}, [x1], #64 + + next_tweak v26, v25, v18, v19 + next_tweak v27, v26, v18, v19 + next_tweak v28, v27, v18, v19 + next_tweak v29, v28, v18, v19 + next_tweak v30, v29, v18, v19 + next_tweak v31, v30, v18, v19 + next_tweak v16, v31, v18, v19 + next_tweak v17, v16, v18, v19 - ld1 {v0.16b}, [x20], #16 - next_tweak v26, v25, v30, v31 eor v0.16b, v0.16b, v25.16b - tbnz x6, #1, 0f - - ld1 {v1.16b}, [x20], #16 - next_tweak v27, v26, v30, v31 eor v1.16b, v1.16b, v26.16b - tbnz x6, #2, 0f - - ld1 {v2.16b}, [x20], #16 - next_tweak v28, v27, v30, v31 eor v2.16b, v2.16b, v27.16b - tbnz x6, #3, 0f - - ld1 {v3.16b}, [x20], #16 - next_tweak v29, v28, v30, v31 eor v3.16b, v3.16b, v28.16b - tbnz x6, #4, 0f - - ld1 {v4.16b}, [x20], #16 - str q29, [sp, #.Lframe_local_offset] eor v4.16b, v4.16b, v29.16b - next_tweak v29, v29, v30, v31 - tbnz x6, #5, 0f - - ld1 {v5.16b}, [x20], #16 - str q29, [sp, #.Lframe_local_offset + 16] - eor v5.16b, v5.16b, v29.16b - next_tweak v29, v29, v30, v31 - tbnz x6, #6, 0f - - ld1 {v6.16b}, [x20], #16 - str q29, [sp, #.Lframe_local_offset + 32] - eor v6.16b, v6.16b, v29.16b - next_tweak v29, v29, v30, v31 - tbnz x6, #7, 0f + eor v5.16b, v5.16b, v30.16b + eor v6.16b, v6.16b, v31.16b + eor v7.16b, v7.16b, v16.16b - ld1 {v7.16b}, [x20], #16 - str q29, [sp, #.Lframe_local_offset + 48] - eor v7.16b, v7.16b, v29.16b - next_tweak v29, v29, v30, v31 + stp q16, q17, [sp, #16] -0: mov bskey, x21 - mov rounds, x22 + mov bskey, x2 + mov rounds, x3 br x16 SYM_FUNC_END(__xts_crypt8) .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7 - frame_push 6, 64 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x5 + stp x29, x30, [sp, #-48]! + mov x29, sp - movi v30.2s, #0x1 - movi v25.2s, #0x87 - uzp1 v30.4s, v30.4s, v25.4s - ld1 {v25.16b}, [x24] + ld1 {v25.16b}, [x5] -99: adr x16, \do8 +0: adr x16, \do8 bl __xts_crypt8 - ldp q16, q17, [sp, #.Lframe_local_offset] - ldp q18, q19, [sp, #.Lframe_local_offset + 32] + eor v16.16b, \o0\().16b, v25.16b + eor v17.16b, \o1\().16b, v26.16b + eor v18.16b, \o2\().16b, v27.16b + eor v19.16b, \o3\().16b, v28.16b - eor \o0\().16b, \o0\().16b, v25.16b - eor \o1\().16b, \o1\().16b, v26.16b - eor \o2\().16b, \o2\().16b, v27.16b - eor \o3\().16b, \o3\().16b, v28.16b + ldp q24, q25, [sp, #16] - st1 {\o0\().16b}, [x19], #16 - mov v25.16b, v26.16b - tbnz x6, #1, 1f - st1 {\o1\().16b}, [x19], #16 - mov v25.16b, v27.16b - tbnz x6, #2, 1f - st1 {\o2\().16b}, [x19], #16 - mov v25.16b, v28.16b - tbnz x6, #3, 1f - st1 {\o3\().16b}, [x19], #16 - mov v25.16b, v29.16b - tbnz x6, #4, 1f + eor v20.16b, \o4\().16b, v29.16b + eor v21.16b, \o5\().16b, v30.16b + eor v22.16b, \o6\().16b, v31.16b + eor v23.16b, \o7\().16b, v24.16b - eor \o4\().16b, \o4\().16b, v16.16b - eor \o5\().16b, \o5\().16b, v17.16b - eor \o6\().16b, \o6\().16b, v18.16b - eor \o7\().16b, \o7\().16b, v19.16b + st1 {v16.16b-v19.16b}, [x0], #64 + st1 {v20.16b-v23.16b}, [x0], #64 - st1 {\o4\().16b}, [x19], #16 - tbnz x6, #5, 1f - st1 {\o5\().16b}, [x19], #16 - tbnz x6, #6, 1f - st1 {\o6\().16b}, [x19], #16 - tbnz x6, #7, 1f - st1 {\o7\().16b}, [x19], #16 + subs x4, x4, #8 + b.gt 0b - cbz x23, 1f - st1 {v25.16b}, [x24] - - b 99b - -1: st1 {v25.16b}, [x24] - frame_pop + st1 {v25.16b}, [x5] + ldp x29, x30, [sp], #48 ret .endm @@ -869,133 +817,51 @@ SYM_FUNC_END(aesbs_xts_decrypt) /* * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], - * int rounds, int blocks, u8 iv[], u8 final[]) + * int rounds, int blocks, u8 iv[]) */ SYM_FUNC_START(aesbs_ctr_encrypt) - frame_push 8 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x5 - mov x25, x6 + stp x29, x30, [sp, #-16]! + mov x29, sp - cmp x25, #0 - cset x26, ne - add x23, x23, x26 // do one extra block if final - - ldp x7, x8, [x24] - ld1 {v0.16b}, [x24] + ldp x7, x8, [x5] + ld1 {v0.16b}, [x5] CPU_LE( rev x7, x7 ) CPU_LE( rev x8, x8 ) adds x8, x8, #1 adc x7, x7, xzr -99: mov x9, #1 - lsl x9, x9, x23 - subs w23, w23, #8 - csel x23, x23, xzr, pl - csel x9, x9, xzr, le - - tbnz x9, #1, 0f - next_ctr v1 - tbnz x9, #2, 0f +0: next_ctr v1 next_ctr v2 - tbnz x9, #3, 0f next_ctr v3 - tbnz x9, #4, 0f next_ctr v4 - tbnz x9, #5, 0f next_ctr v5 - tbnz x9, #6, 0f next_ctr v6 - tbnz x9, #7, 0f next_ctr v7 -0: mov bskey, x21 - mov rounds, x22 + mov bskey, x2 + mov rounds, x3 bl aesbs_encrypt8 - lsr x9, x9, x26 // disregard the extra block - tbnz x9, #0, 0f - - ld1 {v8.16b}, [x20], #16 - eor v0.16b, v0.16b, v8.16b - st1 {v0.16b}, [x19], #16 - tbnz x9, #1, 1f + ld1 { v8.16b-v11.16b}, [x1], #64 + ld1 {v12.16b-v15.16b}, [x1], #64 - ld1 {v9.16b}, [x20], #16 - eor v1.16b, v1.16b, v9.16b - st1 {v1.16b}, [x19], #16 - tbnz x9, #2, 2f + eor v8.16b, v0.16b, v8.16b + eor v9.16b, v1.16b, v9.16b + eor v10.16b, v4.16b, v10.16b + eor v11.16b, v6.16b, v11.16b + eor v12.16b, v3.16b, v12.16b + eor v13.16b, v7.16b, v13.16b + eor v14.16b, v2.16b, v14.16b + eor v15.16b, v5.16b, v15.16b - ld1 {v10.16b}, [x20], #16 - eor v4.16b, v4.16b, v10.16b - st1 {v4.16b}, [x19], #16 - tbnz x9, #3, 3f + st1 { v8.16b-v11.16b}, [x0], #64 + st1 {v12.16b-v15.16b}, [x0], #64 - ld1 {v11.16b}, [x20], #16 - eor v6.16b, v6.16b, v11.16b - st1 {v6.16b}, [x19], #16 - tbnz x9, #4, 4f - - ld1 {v12.16b}, [x20], #16 - eor v3.16b, v3.16b, v12.16b - st1 {v3.16b}, [x19], #16 - tbnz x9, #5, 5f - - ld1 {v13.16b}, [x20], #16 - eor v7.16b, v7.16b, v13.16b - st1 {v7.16b}, [x19], #16 - tbnz x9, #6, 6f + next_ctr v0 + subs x4, x4, #8 + b.gt 0b - ld1 {v14.16b}, [x20], #16 - eor v2.16b, v2.16b, v14.16b - st1 {v2.16b}, [x19], #16 - tbnz x9, #7, 7f - - ld1 {v15.16b}, [x20], #16 - eor v5.16b, v5.16b, v15.16b - st1 {v5.16b}, [x19], #16 - -8: next_ctr v0 - st1 {v0.16b}, [x24] - cbz x23, .Lctr_done - - b 99b - -.Lctr_done: - frame_pop + st1 {v0.16b}, [x5] + ldp x29, x30, [sp], #16 ret - - /* - * If we are handling the tail of the input (x6 != NULL), return the - * final keystream block back to the caller. - */ -0: cbz x25, 8b - st1 {v0.16b}, [x25] - b 8b -1: cbz x25, 8b - st1 {v1.16b}, [x25] - b 8b -2: cbz x25, 8b - st1 {v4.16b}, [x25] - b 8b -3: cbz x25, 8b - st1 {v6.16b}, [x25] - b 8b -4: cbz x25, 8b - st1 {v3.16b}, [x25] - b 8b -5: cbz x25, 8b - st1 {v7.16b}, [x25] - b 8b -6: cbz x25, 8b - st1 {v2.16b}, [x25] - b 8b -7: cbz x25, 8b - st1 {v5.16b}, [x25] - b 8b SYM_FUNC_END(aesbs_ctr_encrypt) diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index 8df6ad8cb09d..bac4cabef607 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -34,7 +34,7 @@ asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], - int rounds, int blocks, u8 iv[], u8 final[]); + int rounds, int blocks, u8 iv[]); asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); @@ -46,6 +46,8 @@ asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int blocks); asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int bytes, u32 const rk2[], u8 iv[], int first); @@ -58,7 +60,7 @@ struct aesbs_ctx { int rounds; } __aligned(AES_BLOCK_SIZE); -struct aesbs_cbc_ctx { +struct aesbs_cbc_ctr_ctx { struct aesbs_ctx key; u32 enc[AES_MAX_KEYLENGTH_U32]; }; @@ -128,10 +130,10 @@ static int ecb_decrypt(struct skcipher_request *req) return __ecb_crypt(req, aesbs_ecb_decrypt); } -static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, +static int aesbs_cbc_ctr_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); struct crypto_aes_ctx rk; int err; @@ -154,7 +156,7 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, static int cbc_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk walk; int err; @@ -177,7 +179,7 @@ static int cbc_encrypt(struct skcipher_request *req) static int cbc_decrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk walk; int err; @@ -205,40 +207,32 @@ static int cbc_decrypt(struct skcipher_request *req) static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk walk; - u8 buf[AES_BLOCK_SIZE]; int err; err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes > 0) { - unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; - u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; - - if (walk.nbytes < walk.total) { - blocks = round_down(blocks, - walk.stride / AES_BLOCK_SIZE); - final = NULL; - } + int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7; + int nbytes = walk.nbytes % (8 * AES_BLOCK_SIZE); + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; kernel_neon_begin(); - aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, - ctx->rk, ctx->rounds, blocks, walk.iv, final); - kernel_neon_end(); - - if (final) { - u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; - u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; - - crypto_xor_cpy(dst, src, final, - walk.total % AES_BLOCK_SIZE); - - err = skcipher_walk_done(&walk, 0); - break; + if (blocks >= 8) { + aesbs_ctr_encrypt(dst, src, ctx->key.rk, ctx->key.rounds, + blocks, walk.iv); + dst += blocks * AES_BLOCK_SIZE; + src += blocks * AES_BLOCK_SIZE; } - err = skcipher_walk_done(&walk, - walk.nbytes - blocks * AES_BLOCK_SIZE); + if (nbytes && walk.nbytes == walk.total) { + neon_aes_ctr_encrypt(dst, src, ctx->enc, ctx->key.rounds, + nbytes, walk.iv); + nbytes = 0; + } + kernel_neon_end(); + err = skcipher_walk_done(&walk, nbytes); } return err; } @@ -308,23 +302,18 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, return err; while (walk.nbytes >= AES_BLOCK_SIZE) { - unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; - - if (walk.nbytes < walk.total || walk.nbytes % AES_BLOCK_SIZE) - blocks = round_down(blocks, - walk.stride / AES_BLOCK_SIZE); - + int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7; out = walk.dst.virt.addr; in = walk.src.virt.addr; nbytes = walk.nbytes; kernel_neon_begin(); - if (likely(blocks > 6)) { /* plain NEON is faster otherwise */ - if (first) + if (blocks >= 8) { + if (first == 1) neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey, ctx->key.rounds, 1); - first = 0; + first = 2; fn(out, in, ctx->key.rk, ctx->key.rounds, blocks, walk.iv); @@ -333,10 +322,17 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, in += blocks * AES_BLOCK_SIZE; nbytes -= blocks * AES_BLOCK_SIZE; } - - if (walk.nbytes == walk.total && nbytes > 0) - goto xts_tail; - + if (walk.nbytes == walk.total && nbytes > 0) { + if (encrypt) + neon_aes_xts_encrypt(out, in, ctx->cts.key_enc, + ctx->key.rounds, nbytes, + ctx->twkey, walk.iv, first); + else + neon_aes_xts_decrypt(out, in, ctx->cts.key_dec, + ctx->key.rounds, nbytes, + ctx->twkey, walk.iv, first); + nbytes = first = 0; + } kernel_neon_end(); err = skcipher_walk_done(&walk, nbytes); } @@ -361,13 +357,12 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, nbytes = walk.nbytes; kernel_neon_begin(); -xts_tail: if (encrypt) neon_aes_xts_encrypt(out, in, ctx->cts.key_enc, ctx->key.rounds, - nbytes, ctx->twkey, walk.iv, first ?: 2); + nbytes, ctx->twkey, walk.iv, first); else neon_aes_xts_decrypt(out, in, ctx->cts.key_dec, ctx->key.rounds, - nbytes, ctx->twkey, walk.iv, first ?: 2); + nbytes, ctx->twkey, walk.iv, first); kernel_neon_end(); return skcipher_walk_done(&walk, 0); @@ -402,14 +397,14 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), + .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctr_ctx), .base.cra_module = THIS_MODULE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .walksize = 8 * AES_BLOCK_SIZE, .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_cbc_setkey, + .setkey = aesbs_cbc_ctr_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { @@ -417,7 +412,7 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctx), + .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctr_ctx), .base.cra_module = THIS_MODULE, .min_keysize = AES_MIN_KEY_SIZE, @@ -425,7 +420,7 @@ static struct skcipher_alg aes_algs[] = { { .chunksize = AES_BLOCK_SIZE, .walksize = 8 * AES_BLOCK_SIZE, .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_setkey, + .setkey = aesbs_cbc_ctr_setkey, .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 8c65cecf560a..250e1377c481 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +// SPDX-License-Identifier: GPL-2.0 /* * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions * diff --git a/arch/arm64/crypto/sha512-armv8.pl b/arch/arm64/crypto/sha512-armv8.pl index 2d8655d5b1af..35ec9ae99fe1 100644 --- a/arch/arm64/crypto/sha512-armv8.pl +++ b/arch/arm64/crypto/sha512-armv8.pl @@ -43,7 +43,7 @@ # on Cortex-A53 (or by 4 cycles per round). # (***) Super-impressive coefficients over gcc-generated code are # indication of some compiler "pathology", most notably code -# generated with -mgeneral-regs-only is significanty faster +# generated with -mgeneral-regs-only is significantly faster # and the gap is only 40-90%. # # October 2016. diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index e62a094a9d52..94cb7580deb7 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +// SPDX-License-Identifier: GPL-2.0 /* * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions * diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c index d71faca322f2..ee98954ae8ca 100644 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ b/arch/arm64/crypto/sm3-ce-glue.c @@ -26,8 +26,10 @@ asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, static int sm3_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) - return crypto_sm3_update(desc, data, len); + if (!crypto_simd_usable()) { + sm3_update(shash_desc_ctx(desc), data, len); + return 0; + } kernel_neon_begin(); sm3_base_do_update(desc, data, len, sm3_ce_transform); @@ -38,8 +40,10 @@ static int sm3_ce_update(struct shash_desc *desc, const u8 *data, static int sm3_ce_final(struct shash_desc *desc, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sm3_finup(desc, NULL, 0, out); + if (!crypto_simd_usable()) { + sm3_final(shash_desc_ctx(desc), out); + return 0; + } kernel_neon_begin(); sm3_base_do_finalize(desc, sm3_ce_transform); @@ -51,14 +55,22 @@ static int sm3_ce_final(struct shash_desc *desc, u8 *out) static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sm3_finup(desc, data, len, out); + if (!crypto_simd_usable()) { + struct sm3_state *sctx = shash_desc_ctx(desc); + + if (len) + sm3_update(sctx, data, len); + sm3_final(sctx, out); + return 0; + } kernel_neon_begin(); - sm3_base_do_update(desc, data, len, sm3_ce_transform); + if (len) + sm3_base_do_update(desc, data, len, sm3_ce_transform); + sm3_base_do_finalize(desc, sm3_ce_transform); kernel_neon_end(); - return sm3_ce_final(desc, out); + return sm3_base_finish(desc, out); } static struct shash_alg sm3_alg = { diff --git a/arch/arm64/include/asm/apple_m1_pmu.h b/arch/arm64/include/asm/apple_m1_pmu.h new file mode 100644 index 000000000000..99483b19b99f --- /dev/null +++ b/arch/arm64/include/asm/apple_m1_pmu.h @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef __ASM_APPLE_M1_PMU_h +#define __ASM_APPLE_M1_PMU_h + +#include <linux/bits.h> +#include <asm/sysreg.h> + +/* Counters */ +#define SYS_IMP_APL_PMC0_EL1 sys_reg(3, 2, 15, 0, 0) +#define SYS_IMP_APL_PMC1_EL1 sys_reg(3, 2, 15, 1, 0) +#define SYS_IMP_APL_PMC2_EL1 sys_reg(3, 2, 15, 2, 0) +#define SYS_IMP_APL_PMC3_EL1 sys_reg(3, 2, 15, 3, 0) +#define SYS_IMP_APL_PMC4_EL1 sys_reg(3, 2, 15, 4, 0) +#define SYS_IMP_APL_PMC5_EL1 sys_reg(3, 2, 15, 5, 0) +#define SYS_IMP_APL_PMC6_EL1 sys_reg(3, 2, 15, 6, 0) +#define SYS_IMP_APL_PMC7_EL1 sys_reg(3, 2, 15, 7, 0) +#define SYS_IMP_APL_PMC8_EL1 sys_reg(3, 2, 15, 9, 0) +#define SYS_IMP_APL_PMC9_EL1 sys_reg(3, 2, 15, 10, 0) + +/* Core PMC control register */ +#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) +#define PMCR0_CNT_ENABLE_0_7 GENMASK(7, 0) +#define PMCR0_IMODE GENMASK(10, 8) +#define PMCR0_IMODE_OFF 0 +#define PMCR0_IMODE_PMI 1 +#define PMCR0_IMODE_AIC 2 +#define PMCR0_IMODE_HALT 3 +#define PMCR0_IMODE_FIQ 4 +#define PMCR0_IACT BIT(11) +#define PMCR0_PMI_ENABLE_0_7 GENMASK(19, 12) +#define PMCR0_STOP_CNT_ON_PMI BIT(20) +#define PMCR0_CNT_GLOB_L2C_EVT BIT(21) +#define PMCR0_DEFER_PMI_TO_ERET BIT(22) +#define PMCR0_ALLOW_CNT_EN_EL0 BIT(30) +#define PMCR0_CNT_ENABLE_8_9 GENMASK(33, 32) +#define PMCR0_PMI_ENABLE_8_9 GENMASK(45, 44) + +#define SYS_IMP_APL_PMCR1_EL1 sys_reg(3, 1, 15, 1, 0) +#define PMCR1_COUNT_A64_EL0_0_7 GENMASK(15, 8) +#define PMCR1_COUNT_A64_EL1_0_7 GENMASK(23, 16) +#define PMCR1_COUNT_A64_EL0_8_9 GENMASK(41, 40) +#define PMCR1_COUNT_A64_EL1_8_9 GENMASK(49, 48) + +#define SYS_IMP_APL_PMCR2_EL1 sys_reg(3, 1, 15, 2, 0) +#define SYS_IMP_APL_PMCR3_EL1 sys_reg(3, 1, 15, 3, 0) +#define SYS_IMP_APL_PMCR4_EL1 sys_reg(3, 1, 15, 4, 0) + +#define SYS_IMP_APL_PMESR0_EL1 sys_reg(3, 1, 15, 5, 0) +#define PMESR0_EVT_CNT_2 GENMASK(7, 0) +#define PMESR0_EVT_CNT_3 GENMASK(15, 8) +#define PMESR0_EVT_CNT_4 GENMASK(23, 16) +#define PMESR0_EVT_CNT_5 GENMASK(31, 24) + +#define SYS_IMP_APL_PMESR1_EL1 sys_reg(3, 1, 15, 6, 0) +#define PMESR1_EVT_CNT_6 GENMASK(7, 0) +#define PMESR1_EVT_CNT_7 GENMASK(15, 8) +#define PMESR1_EVT_CNT_8 GENMASK(23, 16) +#define PMESR1_EVT_CNT_9 GENMASK(31, 24) + +#define SYS_IMP_APL_PMSR_EL1 sys_reg(3, 1, 15, 13, 0) +#define PMSR_OVERFLOW GENMASK(9, 0) + +#endif /* __ASM_APPLE_M1_PMU_h */ diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index 4ad22c3135db..8bd5afc7b692 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -53,17 +53,36 @@ static inline u64 gic_read_iar_common(void) * The gicv3 of ThunderX requires a modified version for reading the * IAR status to ensure data synchronization (access to icc_iar1_el1 * is not sync'ed before and after). + * + * Erratum 38545 + * + * When a IAR register read races with a GIC interrupt RELEASE event, + * GIC-CPU interface could wrongly return a valid INTID to the CPU + * for an interrupt that is already released(non activated) instead of 0x3ff. + * + * To workaround this, return a valid interrupt ID only if there is a change + * in the active priority list after the IAR read. + * + * Common function used for both the workarounds since, + * 1. On Thunderx 88xx 1.x both erratas are applicable. + * 2. Having extra nops doesn't add any side effects for Silicons where + * erratum 23154 is not applicable. */ static inline u64 gic_read_iar_cavium_thunderx(void) { - u64 irqstat; + u64 irqstat, apr; + apr = read_sysreg_s(SYS_ICC_AP1R0_EL1); nops(8); irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1); nops(4); mb(); - return irqstat; + /* Max priority groups implemented is only 32 */ + if (likely(apr != read_sysreg_s(SYS_ICC_AP1R0_EL1))) + return irqstat; + + return 0x3ff; } static inline void gic_write_ctlr(u32 val) diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index 09e43272ccb0..d1bb5e71df25 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -42,13 +42,47 @@ static inline bool __arm64_rndr(unsigned long *v) return ok; } +static inline bool __arm64_rndrrs(unsigned long *v) +{ + bool ok; + + /* + * Reads of RNDRRS set PSTATE.NZCV to 0b0000 on success, + * and set PSTATE.NZCV to 0b0100 otherwise. + */ + asm volatile( + __mrs_s("%0", SYS_RNDRRS_EL0) "\n" + " cset %w1, ne\n" + : "=r" (*v), "=r" (ok) + : + : "cc"); + + return ok; +} + static inline bool __must_check arch_get_random_long(unsigned long *v) { + /* + * Only support the generic interface after we have detected + * the system wide capability, avoiding complexity with the + * cpufeature code and with potential scheduling between CPUs + * with and without the feature. + */ + if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) + return true; return false; } static inline bool __must_check arch_get_random_int(unsigned int *v) { + if (cpus_have_const_cap(ARM64_HAS_RNG)) { + unsigned long val; + + if (__arm64_rndr(&val)) { + *v = val; + return true; + } + } return false; } @@ -71,12 +105,11 @@ static inline bool __must_check arch_get_random_seed_long(unsigned long *v) } /* - * Only support the generic interface after we have detected - * the system wide capability, avoiding complexity with the - * cpufeature code and with potential scheduling between CPUs - * with and without the feature. + * RNDRRS is not backed by an entropy source but by a DRBG that is + * reseeded after each invocation. This is not a 100% fit but good + * enough to implement this API if no other entropy source exists. */ - if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) + if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndrrs(v)) return true; return false; @@ -96,7 +129,7 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) } if (cpus_have_const_cap(ARM64_HAS_RNG)) { - if (__arm64_rndr(&val)) { + if (__arm64_rndrrs(&val)) { *v = val; return true; } diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h index f1bba5fc61c4..ead62f7dd269 100644 --- a/arch/arm64/include/asm/asm_pointer_auth.h +++ b/arch/arm64/include/asm/asm_pointer_auth.h @@ -60,6 +60,9 @@ alternative_else_nop_endif .macro __ptrauth_keys_init_cpu tsk, tmp1, tmp2, tmp3 mrs \tmp1, id_aa64isar1_el1 ubfx \tmp1, \tmp1, #ID_AA64ISAR1_APA_SHIFT, #8 + mrs_s \tmp2, SYS_ID_AA64ISAR2_EL1 + ubfx \tmp2, \tmp2, #ID_AA64ISAR2_APA3_SHIFT, #4 + orr \tmp1, \tmp1, \tmp2 cbz \tmp1, .Lno_addr_auth\@ mov_q \tmp1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index e8bd0af0141c..8c5a61aeaf8e 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -109,6 +109,13 @@ .endm /* + * Clear Branch History instruction + */ + .macro clearbhb + hint #22 + .endm + +/* * Speculation barrier */ .macro sb @@ -535,11 +542,6 @@ alternative_endif #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) #endif -#ifdef CONFIG_KASAN_HW_TAGS -#define EXPORT_SYMBOL_NOHWKASAN(name) -#else -#define EXPORT_SYMBOL_NOHWKASAN(name) EXPORT_SYMBOL_NOKASAN(name) -#endif /* * Emit a 64-bit absolute little endian symbol reference in a way that * ensures that it will be resolved at build time, even when building a @@ -850,4 +852,50 @@ alternative_endif #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ + .macro __mitigate_spectre_bhb_loop tmp +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +alternative_cb spectre_bhb_patch_loop_iter + mov \tmp, #32 // Patched to correct the immediate +alternative_cb_end +.Lspectre_bhb_loop\@: + b . + 4 + subs \tmp, \tmp, #1 + b.ne .Lspectre_bhb_loop\@ + sb +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + + .macro mitigate_spectre_bhb_loop tmp +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +alternative_cb spectre_bhb_patch_loop_mitigation_enable + b .L_spectre_bhb_loop_done\@ // Patched to NOP +alternative_cb_end + __mitigate_spectre_bhb_loop \tmp +.L_spectre_bhb_loop_done\@: +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + + /* Save/restores x0-x3 to the stack */ + .macro __mitigate_spectre_bhb_fw +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 +alternative_cb smccc_patch_fw_mitigation_conduit + nop // Patched to SMC/HVC #0 +alternative_cb_end + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + + .macro mitigate_spectre_bhb_clear_insn +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +alternative_cb spectre_bhb_patch_clearbhb + /* Patched to NOP when not supported */ + clearbhb + isb +alternative_cb_end +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm #endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index ef6be92b1921..c62e7e5e2f0c 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -356,6 +356,7 @@ struct arm64_cpu_capabilities { struct { /* Feature register checking */ u32 sys_reg; u8 field_pos; + u8 field_width; u8 min_field_value; u8 hwcap_type; bool sign; @@ -576,6 +577,8 @@ static inline u64 arm64_ftr_reg_user_value(const struct arm64_ftr_reg *reg) static inline int __attribute_const__ cpuid_feature_extract_field_width(u64 features, int field, int width, bool sign) { + if (WARN_ON_ONCE(!width)) + width = 4; return (sign) ? cpuid_feature_extract_signed_field_width(features, field, width) : cpuid_feature_extract_unsigned_field_width(features, field, width); @@ -637,6 +640,35 @@ static inline bool cpu_supports_mixed_endian_el0(void) return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); } + +static inline bool supports_csv2p3(int scope) +{ + u64 pfr0; + u8 csv2_val; + + if (scope == SCOPE_LOCAL_CPU) + pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); + else + pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + + csv2_val = cpuid_feature_extract_unsigned_field(pfr0, + ID_AA64PFR0_CSV2_SHIFT); + return csv2_val == 3; +} + +static inline bool supports_clearbhb(int scope) +{ + u64 isar2; + + if (scope == SCOPE_LOCAL_CPU) + isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1); + else + isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); + + return cpuid_feature_extract_unsigned_field(isar2, + ID_AA64ISAR2_CLEARBHB_SHIFT); +} + const struct cpumask *system_32bit_el0_cpumask(void); DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); @@ -854,6 +886,7 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) extern struct arm64_ftr_override id_aa64mmfr1_override; extern struct arm64_ftr_override id_aa64pfr1_override; extern struct arm64_ftr_override id_aa64isar1_override; +extern struct arm64_ftr_override id_aa64isar2_override; u32 get_kvm_ipa_limit(void); void dump_cpu_features(void); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 999b9149f856..232b439cbaf3 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -73,10 +73,14 @@ #define ARM_CPU_PART_CORTEX_A76 0xD0B #define ARM_CPU_PART_NEOVERSE_N1 0xD0C #define ARM_CPU_PART_CORTEX_A77 0xD0D +#define ARM_CPU_PART_NEOVERSE_V1 0xD40 +#define ARM_CPU_PART_CORTEX_A78 0xD41 +#define ARM_CPU_PART_CORTEX_X1 0xD44 #define ARM_CPU_PART_CORTEX_A510 0xD46 #define ARM_CPU_PART_CORTEX_A710 0xD47 #define ARM_CPU_PART_CORTEX_X2 0xD48 #define ARM_CPU_PART_NEOVERSE_N2 0xD49 +#define ARM_CPU_PART_CORTEX_A78C 0xD4B #define APM_CPU_PART_POTENZA 0x000 @@ -84,6 +88,13 @@ #define CAVIUM_CPU_PART_THUNDERX_81XX 0x0A2 #define CAVIUM_CPU_PART_THUNDERX_83XX 0x0A3 #define CAVIUM_CPU_PART_THUNDERX2 0x0AF +/* OcteonTx2 series */ +#define CAVIUM_CPU_PART_OCTX2_98XX 0x0B1 +#define CAVIUM_CPU_PART_OCTX2_96XX 0x0B2 +#define CAVIUM_CPU_PART_OCTX2_95XX 0x0B3 +#define CAVIUM_CPU_PART_OCTX2_95XXN 0x0B4 +#define CAVIUM_CPU_PART_OCTX2_95XXMM 0x0B5 +#define CAVIUM_CPU_PART_OCTX2_95XXO 0x0B6 #define BRCM_CPU_PART_BRAHMA_B53 0x100 #define BRCM_CPU_PART_VULCAN 0x516 @@ -117,13 +128,23 @@ #define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) #define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) #define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) +#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1) +#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78) +#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) #define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) +#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) +#define MIDR_OCTX2_98XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_98XX) +#define MIDR_OCTX2_96XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_96XX) +#define MIDR_OCTX2_95XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_95XX) +#define MIDR_OCTX2_95XXN MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_95XXN) +#define MIDR_OCTX2_95XXMM MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_95XXMM) +#define MIDR_OCTX2_95XXO MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_OCTX2_95XXO) #define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2) #define MIDR_BRAHMA_B53 MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_BRAHMA_B53) #define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN) diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h index 657c921fd784..00c291067e57 100644 --- a/arch/arm64/include/asm/debug-monitors.h +++ b/arch/arm64/include/asm/debug-monitors.h @@ -34,18 +34,6 @@ */ #define BREAK_INSTR_SIZE AARCH64_INSN_SIZE -/* - * BRK instruction encoding - * The #imm16 value should be placed at bits[20:5] within BRK ins - */ -#define AARCH64_BREAK_MON 0xd4200000 - -/* - * BRK instruction for provoking a fault on purpose - * Unlike kgdb, #imm16 value with unallocated handler is used for faulting. - */ -#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5)) - #define AARCH64_BREAK_KGDB_DYN_DBG \ (AARCH64_BREAK_MON | (KGDB_DYN_DBG_BRK_IMM << 5)) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 3198acb2aad8..7f3c87f7a0ce 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -106,7 +106,7 @@ msr_s SYS_ICC_SRE_EL2, x0 isb // Make sure SRE is now set mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back, - tbz x0, #0, 1f // and check that it sticks + tbz x0, #0, .Lskip_gicv3_\@ // and check that it sticks msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults .Lskip_gicv3_\@: .endm diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 4335800201c9..daff882883f9 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -62,9 +62,11 @@ enum fixed_addresses { #endif /* CONFIG_ACPI_APEI_GHES */ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + FIX_ENTRY_TRAMP_TEXT3, + FIX_ENTRY_TRAMP_TEXT2, + FIX_ENTRY_TRAMP_TEXT1, FIX_ENTRY_TRAMP_DATA, - FIX_ENTRY_TRAMP_TEXT, -#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT)) +#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1)) #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ __end_of_permanent_fixed_addresses, diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index f68fbb207473..8db5ec0089db 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -108,6 +108,7 @@ #define KERNEL_HWCAP_ECV __khwcap2_feature(ECV) #define KERNEL_HWCAP_AFP __khwcap2_feature(AFP) #define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES) +#define KERNEL_HWCAP_MTE3 __khwcap2_feature(MTE3) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/asm/insn-def.h b/arch/arm64/include/asm/insn-def.h index 2c075f615c6a..1a7d0d483698 100644 --- a/arch/arm64/include/asm/insn-def.h +++ b/arch/arm64/include/asm/insn-def.h @@ -3,7 +3,21 @@ #ifndef __ASM_INSN_DEF_H #define __ASM_INSN_DEF_H +#include <asm/brk-imm.h> + /* A64 instructions are always 32 bits. */ #define AARCH64_INSN_SIZE 4 +/* + * BRK instruction encoding + * The #imm16 value should be placed at bits[20:5] within BRK ins + */ +#define AARCH64_BREAK_MON 0xd4200000 + +/* + * BRK instruction for provoking a fault on purpose + * Unlike kgdb, #imm16 value with unallocated handler is used for faulting. + */ +#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5)) + #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 6b776c8667b2..1e5760d567ae 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op { AARCH64_INSN_HINT_PSB = 0x11 << 5, AARCH64_INSN_HINT_TSB = 0x12 << 5, AARCH64_INSN_HINT_CSDB = 0x14 << 5, + AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5, AARCH64_INSN_HINT_BTI = 0x20 << 5, AARCH64_INSN_HINT_BTIC = 0x22 << 5, @@ -205,7 +206,9 @@ enum aarch64_insn_ldst_type { AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX, AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX, AARCH64_INSN_LDST_LOAD_EX, + AARCH64_INSN_LDST_LOAD_ACQ_EX, AARCH64_INSN_LDST_STORE_EX, + AARCH64_INSN_LDST_STORE_REL_EX, }; enum aarch64_insn_adsb_type { @@ -280,6 +283,36 @@ enum aarch64_insn_adr_type { AARCH64_INSN_ADR_TYPE_ADR, }; +enum aarch64_insn_mem_atomic_op { + AARCH64_INSN_MEM_ATOMIC_ADD, + AARCH64_INSN_MEM_ATOMIC_CLR, + AARCH64_INSN_MEM_ATOMIC_EOR, + AARCH64_INSN_MEM_ATOMIC_SET, + AARCH64_INSN_MEM_ATOMIC_SWP, +}; + +enum aarch64_insn_mem_order_type { + AARCH64_INSN_MEM_ORDER_NONE, + AARCH64_INSN_MEM_ORDER_ACQ, + AARCH64_INSN_MEM_ORDER_REL, + AARCH64_INSN_MEM_ORDER_ACQREL, +}; + +enum aarch64_insn_mb_type { + AARCH64_INSN_MB_SY, + AARCH64_INSN_MB_ST, + AARCH64_INSN_MB_LD, + AARCH64_INSN_MB_ISH, + AARCH64_INSN_MB_ISHST, + AARCH64_INSN_MB_ISHLD, + AARCH64_INSN_MB_NSH, + AARCH64_INSN_MB_NSHST, + AARCH64_INSN_MB_NSHLD, + AARCH64_INSN_MB_OSH, + AARCH64_INSN_MB_OSHST, + AARCH64_INSN_MB_OSHLD, +}; + #define __AARCH64_INSN_FUNCS(abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { \ @@ -303,6 +336,11 @@ __AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400) __AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400) __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800) __AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000) +__AARCH64_INSN_FUNCS(ldclr, 0x3F20FC00, 0x38201000) +__AARCH64_INSN_FUNCS(ldeor, 0x3F20FC00, 0x38202000) +__AARCH64_INSN_FUNCS(ldset, 0x3F20FC00, 0x38203000) +__AARCH64_INSN_FUNCS(swp, 0x3F20FC00, 0x38208000) +__AARCH64_INSN_FUNCS(cas, 0x3FA07C00, 0x08A07C00) __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800) __AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000) __AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) @@ -474,13 +512,6 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg, enum aarch64_insn_register state, enum aarch64_insn_size_type size, enum aarch64_insn_ldst_type type); -u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result, - enum aarch64_insn_register address, - enum aarch64_insn_register value, - enum aarch64_insn_size_type size); -u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address, - enum aarch64_insn_register value, - enum aarch64_insn_size_type size); u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst, enum aarch64_insn_register src, int imm, enum aarch64_insn_variant variant, @@ -541,6 +572,42 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, enum aarch64_insn_prfm_type type, enum aarch64_insn_prfm_target target, enum aarch64_insn_prfm_policy policy); +#ifdef CONFIG_ARM64_LSE_ATOMICS +u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_atomic_op op, + enum aarch64_insn_mem_order_type order); +u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_order_type order); +#else +static inline +u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_atomic_op op, + enum aarch64_insn_mem_order_type order) +{ + return AARCH64_BREAK_FAULT; +} + +static inline +u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_order_type order) +{ + return AARCH64_BREAK_FAULT; +} +#endif +u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type); + s32 aarch64_get_branch_offset(u32 insn); u32 aarch64_set_branch_offset(u32 insn, s32 offset); diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 01d47c5886dc..1767ded83888 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -355,8 +355,8 @@ ECN(SOFTSTP_CUR), ECN(WATCHPT_LOW), ECN(WATCHPT_CUR), \ ECN(BKPT32), ECN(VECTOR32), ECN(BRK64) -#define CPACR_EL1_FPEN (3 << 20) #define CPACR_EL1_TTA (1 << 28) -#define CPACR_EL1_DEFAULT (CPACR_EL1_FPEN | CPACR_EL1_ZEN_EL1EN) +#define CPACR_EL1_DEFAULT (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |\ + CPACR_EL1_ZEN_EL1EN) #endif /* __ARM64_KVM_ARM_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 5bc01e62c08a..031e3a2537fc 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -714,6 +714,11 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr(); } +static inline bool kvm_system_needs_idmapped_vectors(void) +{ + return cpus_have_const_cap(ARM64_SPECTRE_V3A); +} + void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu); static inline void kvm_arch_hardware_unsetup(void) {} diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 462882f356c7..aa7fa2a08f06 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -118,6 +118,7 @@ extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64isar0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64isar1_el1_sys_val); +extern u64 kvm_nvhe_sym(id_aa64isar2_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val); diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index b77e9b3f5371..43f8c25b3fda 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -39,28 +39,4 @@ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ bti c ; -/* - * Annotate a function as position independent, i.e., safe to be called before - * the kernel virtual mapping is activated. - */ -#define SYM_FUNC_START_PI(x) \ - SYM_FUNC_START_ALIAS(__pi_##x); \ - SYM_FUNC_START(x) - -#define SYM_FUNC_START_WEAK_PI(x) \ - SYM_FUNC_START_ALIAS(__pi_##x); \ - SYM_FUNC_START_WEAK(x) - -#define SYM_FUNC_START_WEAK_ALIAS_PI(x) \ - SYM_FUNC_START_ALIAS(__pi_##x); \ - SYM_START(x, SYM_L_WEAK, SYM_A_ALIGN) - -#define SYM_FUNC_END_PI(x) \ - SYM_FUNC_END(x); \ - SYM_FUNC_END_ALIAS(__pi_##x) - -#define SYM_FUNC_END_ALIAS_PI(x) \ - SYM_FUNC_END_ALIAS(x); \ - SYM_FUNC_END_ALIAS(__pi_##x) - #endif diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h index 5d10051c3e62..29c85810ae69 100644 --- a/arch/arm64/include/asm/lse.h +++ b/arch/arm64/include/asm/lse.h @@ -17,12 +17,10 @@ #include <asm/cpucaps.h> extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; -extern struct static_key_false arm64_const_caps_ready; -static inline bool system_uses_lse_atomics(void) +static __always_inline bool system_uses_lse_atomics(void) { - return (static_branch_likely(&arm64_const_caps_ready)) && - static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]); + return static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]); } #define __lse_ll_sc_body(op, ...) \ diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h index a11ccadd47d2..094701ec5500 100644 --- a/arch/arm64/include/asm/module.lds.h +++ b/arch/arm64/include/asm/module.lds.h @@ -1,8 +1,8 @@ SECTIONS { #ifdef CONFIG_ARM64_MODULE_PLTS - .plt 0 (NOLOAD) : { BYTE(0) } - .init.plt 0 (NOLOAD) : { BYTE(0) } - .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) } + .plt 0 : { BYTE(0) } + .init.plt 0 : { BYTE(0) } + .text.ftrace_trampoline 0 : { BYTE(0) } #endif #ifdef CONFIG_KASAN_SW_TAGS diff --git a/arch/arm64/include/asm/mte-def.h b/arch/arm64/include/asm/mte-def.h index 626d359b396e..14ee86b019c2 100644 --- a/arch/arm64/include/asm/mte-def.h +++ b/arch/arm64/include/asm/mte-def.h @@ -11,6 +11,7 @@ #define MTE_TAG_SHIFT 56 #define MTE_TAG_SIZE 4 #define MTE_TAG_MASK GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT) +#define MTE_PAGE_TAG_STORAGE (MTE_GRANULES_PER_PAGE * MTE_TAG_SIZE / 8) #define __MTE_PREAMBLE ARM64_ASM_PREAMBLE ".arch_extension memtag\n" diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index e4704a403237..a857bcacf0fe 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -5,6 +5,7 @@ #ifndef __ASM_MTE_KASAN_H #define __ASM_MTE_KASAN_H +#include <asm/compiler.h> #include <asm/mte-def.h> #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 075539f5f1c8..adcb937342f1 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -11,7 +11,9 @@ #ifndef __ASSEMBLY__ #include <linux/bitfield.h> +#include <linux/kasan-enabled.h> #include <linux/page-flags.h> +#include <linux/sched.h> #include <linux/types.h> #include <asm/pgtable-types.h> @@ -86,6 +88,26 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child, #endif /* CONFIG_ARM64_MTE */ +static inline void mte_disable_tco_entry(struct task_struct *task) +{ + if (!system_supports_mte()) + return; + + /* + * Re-enable tag checking (TCO set on exception entry). This is only + * necessary if MTE is enabled in either the kernel or the userspace + * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set + * for both). With MTE disabled in the kernel and disabled or + * asynchronous in userspace, tag check faults (including in uaccesses) + * are not reported, therefore there is no need to re-enable checking. + * This is beneficial on microarchitectures where re-enabling TCO is + * expensive. + */ + if (kasan_hw_tags_enabled() || + (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT))) + asm volatile(SET_PSTATE_TCO(0)); +} + #ifdef CONFIG_KASAN_HW_TAGS /* Whether the MTE asynchronous mode is enabled. */ DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); diff --git a/arch/arm64/include/asm/paravirt_api_clock.h b/arch/arm64/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..65ac7cee0dad --- /dev/null +++ b/arch/arm64/include/asm/paravirt_api_clock.h @@ -0,0 +1 @@ +#include <asm/paravirt.h> diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 4ef6f19331f9..3eaf462f5752 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -15,70 +15,70 @@ /* * Common architectural and microarchitectural event numbers. */ -#define ARMV8_PMUV3_PERFCTR_SW_INCR 0x00 -#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x01 -#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x02 -#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL 0x03 -#define ARMV8_PMUV3_PERFCTR_L1D_CACHE 0x04 -#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x05 -#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x06 -#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x07 -#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x08 -#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x09 -#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x0A -#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x0B -#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x0C -#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x0D -#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x0E -#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x0F -#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED 0x10 -#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES 0x11 -#define ARMV8_PMUV3_PERFCTR_BR_PRED 0x12 -#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x13 -#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x14 -#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x15 -#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x16 -#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x17 -#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x18 -#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x19 -#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x1A -#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x1B -#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x1C -#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x1D -#define ARMV8_PMUV3_PERFCTR_CHAIN 0x1E -#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x1F -#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x20 -#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x21 -#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x22 -#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x23 -#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x24 -#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x25 -#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x26 -#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x27 -#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x28 -#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x29 -#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x2A -#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x2B -#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x2C -#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x2D -#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x2E -#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x2F -#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x30 -#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS 0x31 -#define ARMV8_PMUV3_PERFCTR_LL_CACHE 0x32 -#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS 0x33 -#define ARMV8_PMUV3_PERFCTR_DTLB_WALK 0x34 -#define ARMV8_PMUV3_PERFCTR_ITLB_WALK 0x35 -#define ARMV8_PMUV3_PERFCTR_LL_CACHE_RD 0x36 -#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD 0x37 -#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD 0x38 -#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD 0x39 -#define ARMV8_PMUV3_PERFCTR_OP_RETIRED 0x3A -#define ARMV8_PMUV3_PERFCTR_OP_SPEC 0x3B -#define ARMV8_PMUV3_PERFCTR_STALL 0x3C -#define ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND 0x3D -#define ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND 0x3E -#define ARMV8_PMUV3_PERFCTR_STALL_SLOT 0x3F +#define ARMV8_PMUV3_PERFCTR_SW_INCR 0x0000 +#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x0001 +#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x0002 +#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL 0x0003 +#define ARMV8_PMUV3_PERFCTR_L1D_CACHE 0x0004 +#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x0005 +#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x0006 +#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x0007 +#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x0008 +#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x0009 +#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x000A +#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x000B +#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x000C +#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x000D +#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x000E +#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x000F +#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED 0x0010 +#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES 0x0011 +#define ARMV8_PMUV3_PERFCTR_BR_PRED 0x0012 +#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x0013 +#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x0014 +#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x0015 +#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x0016 +#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x0017 +#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x0018 +#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x0019 +#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x001A +#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x001B +#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x001C +#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x001D +#define ARMV8_PMUV3_PERFCTR_CHAIN 0x001E +#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x001F +#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x0020 +#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x0021 +#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x0022 +#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x0023 +#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x0024 +#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x0025 +#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x0026 +#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x0027 +#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x0028 +#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x0029 +#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x002A +#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x002B +#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x002C +#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x002D +#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x002E +#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x002F +#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x0030 +#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS 0x0031 +#define ARMV8_PMUV3_PERFCTR_LL_CACHE 0x0032 +#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS 0x0033 +#define ARMV8_PMUV3_PERFCTR_DTLB_WALK 0x0034 +#define ARMV8_PMUV3_PERFCTR_ITLB_WALK 0x0035 +#define ARMV8_PMUV3_PERFCTR_LL_CACHE_RD 0x0036 +#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD 0x0037 +#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD 0x0038 +#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD 0x0039 +#define ARMV8_PMUV3_PERFCTR_OP_RETIRED 0x003A +#define ARMV8_PMUV3_PERFCTR_OP_SPEC 0x003B +#define ARMV8_PMUV3_PERFCTR_STALL 0x003C +#define ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND 0x003D +#define ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND 0x003E +#define ARMV8_PMUV3_PERFCTR_STALL_SLOT 0x003F /* Statistical profiling extension microarchitectural events */ #define ARMV8_SPE_PERFCTR_SAMPLE_POP 0x4000 @@ -96,6 +96,20 @@ #define ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS 0x400A #define ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD 0x400B +/* Trace buffer events */ +#define ARMV8_PMUV3_PERFCTR_TRB_WRAP 0x400C +#define ARMV8_PMUV3_PERFCTR_TRB_TRIG 0x400E + +/* Trace unit events */ +#define ARMV8_PMUV3_PERFCTR_TRCEXTOUT0 0x4010 +#define ARMV8_PMUV3_PERFCTR_TRCEXTOUT1 0x4011 +#define ARMV8_PMUV3_PERFCTR_TRCEXTOUT2 0x4012 +#define ARMV8_PMUV3_PERFCTR_TRCEXTOUT3 0x4013 +#define ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT4 0x4018 +#define ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT5 0x4019 +#define ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT6 0x401A +#define ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT7 0x401B + /* additional latency from alignment events */ #define ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT 0x4020 #define ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT 0x4021 @@ -107,91 +121,91 @@ #define ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_WR 0x4026 /* ARMv8 recommended implementation defined event types */ -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD 0x40 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR 0x41 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD 0x42 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR 0x43 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER 0x44 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER 0x45 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM 0x46 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN 0x47 -#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL 0x48 - -#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD 0x4C -#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR 0x4D -#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD 0x4E -#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR 0x4F -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD 0x50 -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR 0x51 -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD 0x52 -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR 0x53 - -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM 0x56 -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN 0x57 -#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL 0x58 - -#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD 0x5C -#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR 0x5D -#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD 0x5E -#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR 0x5F -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD 0x60 -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR 0x61 -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED 0x62 -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED 0x63 -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL 0x64 -#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH 0x65 -#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD 0x66 -#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR 0x67 -#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC 0x68 -#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC 0x69 -#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC 0x6A - -#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC 0x6C -#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC 0x6D -#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC 0x6E -#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC 0x6F -#define ARMV8_IMPDEF_PERFCTR_LD_SPEC 0x70 -#define ARMV8_IMPDEF_PERFCTR_ST_SPEC 0x71 -#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC 0x72 -#define ARMV8_IMPDEF_PERFCTR_DP_SPEC 0x73 -#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC 0x74 -#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC 0x75 -#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC 0x76 -#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC 0x77 -#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC 0x78 -#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC 0x79 -#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC 0x7A - -#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC 0x7C -#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC 0x7D -#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC 0x7E - -#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF 0x81 -#define ARMV8_IMPDEF_PERFCTR_EXC_SVC 0x82 -#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT 0x83 -#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT 0x84 - -#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ 0x86 -#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ 0x87 -#define ARMV8_IMPDEF_PERFCTR_EXC_SMC 0x88 - -#define ARMV8_IMPDEF_PERFCTR_EXC_HVC 0x8A -#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT 0x8B -#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT 0x8C -#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER 0x8D -#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ 0x8E -#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ 0x8F -#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC 0x90 -#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC 0x91 - -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD 0xA0 -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR 0xA1 -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD 0xA2 -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR 0xA3 - -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM 0xA6 -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN 0xA7 -#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL 0xA8 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD 0x0040 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR 0x0041 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD 0x0042 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR 0x0043 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER 0x0044 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER 0x0045 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM 0x0046 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN 0x0047 +#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL 0x0048 + +#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD 0x004C +#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR 0x004D +#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD 0x004E +#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR 0x004F +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD 0x0050 +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR 0x0051 +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD 0x0052 +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR 0x0053 + +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM 0x0056 +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN 0x0057 +#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL 0x0058 + +#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD 0x005C +#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR 0x005D +#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD 0x005E +#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR 0x005F +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD 0x0060 +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR 0x0061 +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED 0x0062 +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED 0x0063 +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL 0x0064 +#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH 0x0065 +#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD 0x0066 +#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR 0x0067 +#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC 0x0068 +#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC 0x0069 +#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC 0x006A + +#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC 0x006C +#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC 0x006D +#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC 0x006E +#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC 0x006F +#define ARMV8_IMPDEF_PERFCTR_LD_SPEC 0x0070 +#define ARMV8_IMPDEF_PERFCTR_ST_SPEC 0x0071 +#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC 0x0072 +#define ARMV8_IMPDEF_PERFCTR_DP_SPEC 0x0073 +#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC 0x0074 +#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC 0x0075 +#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC 0x0076 +#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC 0x0077 +#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC 0x0078 +#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC 0x0079 +#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC 0x007A + +#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC 0x007C +#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC 0x007D +#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC 0x007E + +#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF 0x0081 +#define ARMV8_IMPDEF_PERFCTR_EXC_SVC 0x0082 +#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT 0x0083 +#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT 0x0084 + +#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ 0x0086 +#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ 0x0087 +#define ARMV8_IMPDEF_PERFCTR_EXC_SMC 0x0088 + +#define ARMV8_IMPDEF_PERFCTR_EXC_HVC 0x008A +#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT 0x008B +#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT 0x008C +#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER 0x008D +#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ 0x008E +#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ 0x008F +#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC 0x0090 +#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC 0x0091 + +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD 0x00A0 +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR 0x00A1 +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD 0x00A2 +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR 0x00A3 + +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM 0x00A6 +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN 0x00A7 +#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL 0x00A8 /* * Per-CPU PMCR: config reg diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 40085e53f573..66671ff05183 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -273,6 +273,8 @@ #define TCR_NFD1 (UL(1) << 54) #define TCR_E0PD0 (UL(1) << 55) #define TCR_E0PD1 (UL(1) << 56) +#define TCR_TCMA0 (UL(1) << 57) +#define TCR_TCMA1 (UL(1) << 58) /* * TTBR. diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 7032f04c8ac6..b1e1b74d993c 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -92,7 +92,7 @@ extern bool arm64_use_ng_mappings; #define __P001 PAGE_READONLY #define __P010 PAGE_READONLY #define __P011 PAGE_READONLY -#define __P100 PAGE_EXECONLY +#define __P100 PAGE_READONLY_EXEC /* PAGE_EXECONLY if Enhanced PAN */ #define __P101 PAGE_READONLY_EXEC #define __P110 PAGE_READONLY_EXEC #define __P111 PAGE_READONLY_EXEC @@ -101,7 +101,7 @@ extern bool arm64_use_ng_mappings; #define __S001 PAGE_READONLY #define __S010 PAGE_SHARED #define __S011 PAGE_SHARED -#define __S100 PAGE_EXECONLY +#define __S100 PAGE_READONLY_EXEC /* PAGE_EXECONLY if Enhanced PAN */ #define __S101 PAGE_READONLY_EXEC #define __S110 PAGE_SHARED_EXEC #define __S111 PAGE_SHARED_EXEC diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index c4ba047a82d2..94e147e5456c 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1017,17 +1017,6 @@ static inline bool arch_wants_old_prefaulted_pte(void) } #define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte -static inline pgprot_t arch_filter_pgprot(pgprot_t prot) -{ - if (cpus_have_const_cap(ARM64_HAS_EPAN)) - return prot; - - if (pgprot_val(prot) != pgprot_val(PAGE_EXECONLY)) - return prot; - - return PAGE_READONLY_EXEC; -} - static inline bool pud_sect_supported(void) { return PAGE_SIZE == SZ_4K; diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index e83f0982b99c..0159b625cc7f 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -2,6 +2,7 @@ #ifndef __ASM_PREEMPT_H #define __ASM_PREEMPT_H +#include <linux/jump_label.h> #include <linux/thread_info.h> #define PREEMPT_NEED_RESCHED BIT(32) @@ -80,10 +81,24 @@ static inline bool should_resched(int preempt_offset) } #ifdef CONFIG_PREEMPTION + void preempt_schedule(void); -#define __preempt_schedule() preempt_schedule() void preempt_schedule_notrace(void); -#define __preempt_schedule_notrace() preempt_schedule_notrace() + +#ifdef CONFIG_PREEMPT_DYNAMIC + +DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +void dynamic_preempt_schedule(void); +#define __preempt_schedule() dynamic_preempt_schedule() +void dynamic_preempt_schedule_notrace(void); +#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() + +#else /* CONFIG_PREEMPT_DYNAMIC */ + +#define __preempt_schedule() preempt_schedule() +#define __preempt_schedule_notrace() preempt_schedule_notrace() + +#endif /* CONFIG_PREEMPT_DYNAMIC */ #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 6f41b65f9962..73e38d9a540c 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -21,6 +21,7 @@ #define MTE_CTRL_TCF_SYNC (1UL << 16) #define MTE_CTRL_TCF_ASYNC (1UL << 17) +#define MTE_CTRL_TCF_ASYMM (1UL << 18) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h index 1bce62fa908a..56f7b1d4d54b 100644 --- a/arch/arm64/include/asm/rwonce.h +++ b/arch/arm64/include/asm/rwonce.h @@ -5,7 +5,7 @@ #ifndef __ASM_RWONCE_H #define __ASM_RWONCE_H -#ifdef CONFIG_LTO +#if defined(CONFIG_LTO) && !defined(__ASSEMBLY__) #include <linux/compiler_types.h> #include <asm/alternative-macros.h> @@ -66,7 +66,7 @@ }) #endif /* !BUILD_VDSO */ -#endif /* CONFIG_LTO */ +#endif /* CONFIG_LTO && !__ASSEMBLY__ */ #include <asm-generic/rwonce.h> diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index 152cb35bf9df..40971ac1303f 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -23,4 +23,9 @@ extern char __mmuoff_data_start[], __mmuoff_data_end[]; extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[]; +static inline size_t entry_tramp_text_size(void) +{ + return __entry_tramp_text_end - __entry_tramp_text_start; +} + #endif /* __ASM_SECTIONS_H */ diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index f62ca39da6c5..aa3d3607d5c8 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -67,7 +67,8 @@ struct bp_hardening_data { DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); -static inline void arm64_apply_bp_hardening(void) +/* Called during entry so must be __always_inline */ +static __always_inline void arm64_apply_bp_hardening(void) { struct bp_hardening_data *d; @@ -93,5 +94,9 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk); enum mitigation_state arm64_get_meltdown_state(void); +enum mitigation_state arm64_get_spectre_bhb_state(void); +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); +u8 spectre_bhb_loop_affected(int scope); +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); #endif /* __ASSEMBLY__ */ #endif /* __ASM_SPECTRE_H */ diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h index 95f7686b728d..3a3264ff47b9 100644 --- a/arch/arm64/include/asm/string.h +++ b/arch/arm64/include/asm/string.h @@ -12,13 +12,11 @@ extern char *strrchr(const char *, int c); #define __HAVE_ARCH_STRCHR extern char *strchr(const char *, int c); -#ifndef CONFIG_KASAN_HW_TAGS #define __HAVE_ARCH_STRCMP extern int strcmp(const char *, const char *); #define __HAVE_ARCH_STRNCMP extern int strncmp(const char *, const char *, __kernel_size_t); -#endif #define __HAVE_ARCH_STRLEN extern __kernel_size_t strlen(const char *); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 898bee0004ae..c7ca3a105528 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -773,6 +773,9 @@ #define ID_AA64ISAR1_GPI_IMP_DEF 0x1 /* id_aa64isar2 */ +#define ID_AA64ISAR2_CLEARBHB_SHIFT 28 +#define ID_AA64ISAR2_APA3_SHIFT 12 +#define ID_AA64ISAR2_GPA3_SHIFT 8 #define ID_AA64ISAR2_RPRES_SHIFT 4 #define ID_AA64ISAR2_WFXT_SHIFT 0 @@ -786,6 +789,16 @@ #define ID_AA64ISAR2_WFXT_NI 0x0 #define ID_AA64ISAR2_WFXT_SUPPORTED 0x2 +#define ID_AA64ISAR2_APA3_NI 0x0 +#define ID_AA64ISAR2_APA3_ARCHITECTED 0x1 +#define ID_AA64ISAR2_APA3_ARCH_EPAC 0x2 +#define ID_AA64ISAR2_APA3_ARCH_EPAC2 0x3 +#define ID_AA64ISAR2_APA3_ARCH_EPAC2_FPAC 0x4 +#define ID_AA64ISAR2_APA3_ARCH_EPAC2_FPAC_CMB 0x5 + +#define ID_AA64ISAR2_GPA3_NI 0x0 +#define ID_AA64ISAR2_GPA3_ARCHITECTED 0x1 + /* id_aa64pfr0 */ #define ID_AA64PFR0_CSV3_SHIFT 60 #define ID_AA64PFR0_CSV2_SHIFT 56 @@ -904,6 +917,7 @@ #endif /* id_aa64mmfr1 */ +#define ID_AA64MMFR1_ECBHB_SHIFT 60 #define ID_AA64MMFR1_AFP_SHIFT 44 #define ID_AA64MMFR1_ETS_SHIFT 36 #define ID_AA64MMFR1_TWED_SHIFT 32 @@ -1097,13 +1111,11 @@ #define ZCR_ELx_LEN_SIZE 9 #define ZCR_ELx_LEN_MASK 0x1ff +#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */ +#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */ + #define CPACR_EL1_ZEN_EL1EN (BIT(16)) /* enable EL1 access */ #define CPACR_EL1_ZEN_EL0EN (BIT(17)) /* enable EL0 access, if EL1EN set */ -#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN) - -/* TCR EL1 Bit Definitions */ -#define SYS_TCR_EL1_TCMA1 (BIT(58)) -#define SYS_TCR_EL1_TCMA0 (BIT(57)) /* GCR_EL1 Definitions */ #define SYS_GCR_EL1_RRND (BIT(16)) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index f386b90a79c8..9fab663dd2de 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -24,6 +24,10 @@ void update_freq_counters_refs(void); #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#ifdef CONFIG_ACPI_CPPC_LIB +#define arch_init_invariance_cppc topology_init_cpu_capacity_cppc +#endif + /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 3a5ff5e20586..e8dce0cc5eaa 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -26,7 +26,7 @@ #include <asm/memory.h> #include <asm/extable.h> -#define HAVE_GET_KERNEL_NOFAULT +static inline int __access_ok(const void __user *ptr, unsigned long size); /* * Test whether a block of memory is a valid user space address. @@ -35,10 +35,8 @@ * This is equivalent to the following test: * (u65)addr + (u65)size <= (u65)TASK_SIZE_MAX */ -static inline unsigned long __range_ok(const void __user *addr, unsigned long size) +static inline int access_ok(const void __user *addr, unsigned long size) { - unsigned long ret, limit = TASK_SIZE_MAX - 1; - /* * Asynchronous I/O running in a kernel thread does not have the * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag @@ -48,28 +46,11 @@ static inline unsigned long __range_ok(const void __user *addr, unsigned long si (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) addr = untagged_addr(addr); - __chk_user_ptr(addr); - asm volatile( - // A + B <= C + 1 for all A,B,C, in four easy steps: - // 1: X = A + B; X' = X % 2^64 - " adds %0, %3, %2\n" - // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4 - " csel %1, xzr, %1, hi\n" - // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X' - // to compensate for the carry flag being set in step 4. For - // X > 2^64, X' merely has to remain nonzero, which it does. - " csinv %0, %0, xzr, cc\n" - // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1 - // comes from the carry in being clear. Otherwise, we are - // testing X' - C == 0, subject to the previous adjustments. - " sbcs xzr, %0, %1\n" - " cset %0, ls\n" - : "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc"); - - return ret; + return likely(__access_ok(addr, size)); } +#define access_ok access_ok -#define access_ok(addr, size) __range_ok(addr, size) +#include <asm-generic/access_ok.h> /* * User access enabling/disabling. diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h new file mode 100644 index 000000000000..bc9a2145f419 --- /dev/null +++ b/arch/arm64/include/asm/vectors.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 ARM Ltd. + */ +#ifndef __ASM_VECTORS_H +#define __ASM_VECTORS_H + +#include <linux/bug.h> +#include <linux/percpu.h> + +#include <asm/fixmap.h> + +extern char vectors[]; +extern char tramp_vectors[]; +extern char __bp_harden_el1_vectors[]; + +/* + * Note: the order of this enum corresponds to two arrays in entry.S: + * tramp_vecs and __bp_harden_el1_vectors. By default the canonical + * 'full fat' vectors are used directly. + */ +enum arm64_bp_harden_el1_vectors { +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + /* + * Perform the BHB loop mitigation, before branching to the canonical + * vectors. + */ + EL1_VECTOR_BHB_LOOP, + + /* + * Make the SMC call for firmware mitigation, before branching to the + * canonical vectors. + */ + EL1_VECTOR_BHB_FW, + + /* + * Use the ClearBHB instruction, before branching to the canonical + * vectors. + */ + EL1_VECTOR_BHB_CLEAR_INSN, +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + + /* + * Remap the kernel before branching to the canonical vectors. + */ + EL1_VECTOR_KPTI, +}; + +#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +#define EL1_VECTOR_BHB_LOOP -1 +#define EL1_VECTOR_BHB_FW -1 +#define EL1_VECTOR_BHB_CLEAR_INSN -1 +#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + +/* The vectors to use on return from EL0. e.g. to remap the kernel */ +DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); + +#ifndef CONFIG_UNMAP_KERNEL_AT_EL0 +#define TRAMP_VALIAS 0ul +#endif + +static inline const char * +arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot) +{ + if (arm64_kernel_unmapped_at_el0()) + return (char *)(TRAMP_VALIAS + SZ_2K * slot); + + WARN_ON_ONCE(slot == EL1_VECTOR_KPTI); + + return __bp_harden_el1_vectors + SZ_2K * slot; +} + +#endif /* __ASM_VECTORS_H */ diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h index 947f6a4f1aa0..befcd8a7abc9 100644 --- a/arch/arm64/include/asm/xor.h +++ b/arch/arm64/include/asm/xor.h @@ -16,7 +16,8 @@ extern struct xor_block_template const xor_block_inner_neon; static void -xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { kernel_neon_begin(); xor_block_inner_neon.do_2(bytes, p1, p2); @@ -24,8 +25,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { kernel_neon_begin(); xor_block_inner_neon.do_3(bytes, p1, p2, p3); @@ -33,8 +35,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { kernel_neon_begin(); xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4); @@ -42,8 +46,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { kernel_neon_begin(); xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5); diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index f03731847d9d..99cb5d383048 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -78,5 +78,6 @@ #define HWCAP2_ECV (1 << 19) #define HWCAP2_AFP (1 << 20) #define HWCAP2_RPRES (1 << 21) +#define HWCAP2_MTE3 (1 << 22) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index b3edde68bc3e..323e251ed37b 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -281,6 +281,11 @@ struct kvm_arm_copy_mte_tags { #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3 #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4) +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 KVM_REG_ARM_FW_REG(3) +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL 0 +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL 1 +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED 2 + /* SVE registers */ #define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT) diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 88b3e2a21408..986837d7ec82 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -61,6 +61,7 @@ obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o obj-$(CONFIG_PARAVIRT) += paravirt.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o +obj-$(CONFIG_ELF_CORE) += elfcore.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ cpu-reset.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index b217941713a8..4c9b5b4b7a0b 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -214,6 +214,21 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { }; #endif +#ifdef CONFIG_CAVIUM_ERRATUM_23154 +const struct midr_range cavium_erratum_23154_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_THUNDERX), + MIDR_ALL_VERSIONS(MIDR_THUNDERX_81XX), + MIDR_ALL_VERSIONS(MIDR_THUNDERX_83XX), + MIDR_ALL_VERSIONS(MIDR_OCTX2_98XX), + MIDR_ALL_VERSIONS(MIDR_OCTX2_96XX), + MIDR_ALL_VERSIONS(MIDR_OCTX2_95XX), + MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXN), + MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXMM), + MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXO), + {}, +}; +#endif + #ifdef CONFIG_CAVIUM_ERRATUM_27456 const struct midr_range cavium_erratum_27456_cpus[] = { /* Cavium ThunderX, T88 pass 1.x - 2.1 */ @@ -425,10 +440,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_CAVIUM_ERRATUM_23154 { - /* Cavium ThunderX, pass 1.x */ - .desc = "Cavium erratum 23154", + .desc = "Cavium errata 23154 and 38545", .capability = ARM64_WORKAROUND_CAVIUM_23154, - ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1), + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + ERRATA_MIDR_RANGE_LIST(cavium_erratum_23154_cpus), }, #endif #ifdef CONFIG_CAVIUM_ERRATUM_27456 @@ -502,6 +517,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = has_spectre_v4, .cpu_enable = spectre_v4_enable_mitigation, }, + { + .desc = "Spectre-BHB", + .capability = ARM64_SPECTRE_BHB, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = is_spectre_bhb_affected, + .cpu_enable = spectre_bhb_enable_mitigation, + }, #ifdef CONFIG_ARM64_ERRATUM_1418040 { .desc = "ARM erratum 1418040", @@ -604,7 +626,6 @@ const struct arm64_cpu_capabilities arm64_errata[] = { { .desc = "ARM erratum 2077057", .capability = ARM64_WORKAROUND_2077057, - .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2), }, #endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e5f23dab1c8d..d72c4b4d389c 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -73,6 +73,8 @@ #include <linux/mm.h> #include <linux/cpu.h> #include <linux/kasan.h> +#include <linux/percpu.h> + #include <asm/cpu.h> #include <asm/cpufeature.h> #include <asm/cpu_ops.h> @@ -85,6 +87,7 @@ #include <asm/smp.h> #include <asm/sysreg.h> #include <asm/traps.h> +#include <asm/vectors.h> #include <asm/virt.h> /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ @@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); bool arm64_use_ng_mappings = false; EXPORT_SYMBOL(arm64_use_ng_mappings); +DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors; + /* * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs * support it? @@ -226,6 +231,11 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), + FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_APA3_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_GPA3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -596,6 +606,7 @@ static const struct arm64_ftr_bits ftr_raz[] = { struct arm64_ftr_override __ro_after_init id_aa64mmfr1_override; struct arm64_ftr_override __ro_after_init id_aa64pfr1_override; struct arm64_ftr_override __ro_after_init id_aa64isar1_override; +struct arm64_ftr_override __ro_after_init id_aa64isar2_override; static const struct __ftr_reg_entry { u32 sys_id; @@ -644,6 +655,8 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1, &id_aa64isar1_override), ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2), + ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2, + &id_aa64isar2_override), /* Op1 = 0, CRn = 0, CRm = 7 */ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), @@ -1307,7 +1320,9 @@ u64 __read_sysreg_by_encoding(u32 sys_id) static bool feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { - int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign); + int val = cpuid_feature_extract_field_width(reg, entry->field_pos, + entry->field_width, + entry->sign); return val >= entry->min_field_value; } @@ -1590,6 +1605,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) int cpu = smp_processor_id(); + if (__this_cpu_read(this_cpu_vector) == vectors) { + const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); + + __this_cpu_write(this_cpu_vector, v); + } + /* * We don't need to rewrite the page-tables if either we've done * it already or we have KASLR enabled and therefore have not @@ -1775,14 +1796,6 @@ static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused) write_sysreg(read_sysreg(tpidr_el1), tpidr_el2); } -static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused) -{ - u64 val = read_sysreg_s(SYS_CLIDR_EL1); - - /* Check that CLIDR_EL1.LOU{U,IS} are both 0 */ - WARN_ON(CLIDR_LOUU(val) || CLIDR_LOUIS(val)); -} - #ifdef CONFIG_ARM64_PAN static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) { @@ -1829,21 +1842,27 @@ static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, /* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */ sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg), entry->field_pos, entry->sign); - return sec_val == boot_val; + return (sec_val >= entry->min_field_value) && (sec_val == boot_val); } static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry, int scope) { - return has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH], scope) || - has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope); + bool api = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope); + bool apa = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope); + bool apa3 = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope); + + return apa || apa3 || api; } static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, int __unused) { - return __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH) || - __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF); + bool gpi = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF); + bool gpa = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5); + bool gpa3 = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3); + + return gpa || gpa3 || gpi; } #endif /* CONFIG_ARM64_PTR_AUTH */ @@ -1945,6 +1964,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_useable_gicv3_cpuif, .sys_reg = SYS_ID_AA64PFR0_EL1, .field_pos = ID_AA64PFR0_GIC_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, }, @@ -1955,6 +1975,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR0_EL1, .field_pos = ID_AA64MMFR0_ECV_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, }, @@ -1966,6 +1987,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR1_EL1, .field_pos = ID_AA64MMFR1_PAN_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, .cpu_enable = cpu_enable_pan, @@ -1979,6 +2001,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR1_EL1, .field_pos = ID_AA64MMFR1_PAN_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 3, }, @@ -1991,6 +2014,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR0_EL1, .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 2, }, @@ -2015,6 +2039,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_EL0_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT, }, #ifdef CONFIG_KVM @@ -2026,6 +2051,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_EL1_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT, }, { @@ -2046,6 +2072,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { */ .sys_reg = SYS_ID_AA64PFR0_EL1, .field_pos = ID_AA64PFR0_CSV3_SHIFT, + .field_width = 4, .min_field_value = 1, .matches = unmap_kernel_at_el0, .cpu_enable = kpti_install_ng_mappings, @@ -2065,6 +2092,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR1_EL1, .field_pos = ID_AA64ISAR1_DPB_SHIFT, + .field_width = 4, .min_field_value = 1, }, { @@ -2075,6 +2103,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_DPB_SHIFT, + .field_width = 4, .min_field_value = 2, }, #endif @@ -2086,6 +2115,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_SVE_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR0_SVE, .matches = has_cpuid_feature, .cpu_enable = sve_kernel_enable, @@ -2100,6 +2130,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_RAS_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR0_RAS_V1, .cpu_enable = cpu_clear_disr, }, @@ -2118,6 +2149,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_AMU_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR0_AMU, .cpu_enable = cpu_amu_enable, }, @@ -2142,9 +2174,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64MMFR2_FWB_SHIFT, + .field_width = 4, .min_field_value = 1, .matches = has_cpuid_feature, - .cpu_enable = cpu_has_fwb, }, { .desc = "ARMv8.4 Translation Table Level", @@ -2153,6 +2185,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64MMFR2_TTL_SHIFT, + .field_width = 4, .min_field_value = 1, .matches = has_cpuid_feature, }, @@ -2163,6 +2196,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR0_EL1, .field_pos = ID_AA64ISAR0_TLB_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = ID_AA64ISAR0_TLB_RANGE, }, @@ -2181,6 +2215,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64MMFR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64MMFR1_HADBS_SHIFT, + .field_width = 4, .min_field_value = 2, .matches = has_hw_dbm, .cpu_enable = cpu_enable_hw_dbm, @@ -2193,6 +2228,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR0_EL1, .field_pos = ID_AA64ISAR0_CRC32_SHIFT, + .field_width = 4, .min_field_value = 1, }, { @@ -2202,6 +2238,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, .field_pos = ID_AA64PFR1_SSBS_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY, }, @@ -2214,6 +2251,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64MMFR2_CNP_SHIFT, + .field_width = 4, .min_field_value = 1, .cpu_enable = cpu_enable_cnp, }, @@ -2225,27 +2263,41 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR1_EL1, .field_pos = ID_AA64ISAR1_SB_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, }, #ifdef CONFIG_ARM64_PTR_AUTH { - .desc = "Address authentication (architected algorithm)", - .capability = ARM64_HAS_ADDRESS_AUTH_ARCH, + .desc = "Address authentication (architected QARMA5 algorithm)", + .capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_APA_SHIFT, + .field_width = 4, .min_field_value = ID_AA64ISAR1_APA_ARCHITECTED, .matches = has_address_auth_cpucap, }, { + .desc = "Address authentication (architected QARMA3 algorithm)", + .capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, + .sys_reg = SYS_ID_AA64ISAR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64ISAR2_APA3_SHIFT, + .field_width = 4, + .min_field_value = ID_AA64ISAR2_APA3_ARCHITECTED, + .matches = has_address_auth_cpucap, + }, + { .desc = "Address authentication (IMP DEF algorithm)", .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_API_SHIFT, + .field_width = 4, .min_field_value = ID_AA64ISAR1_API_IMP_DEF, .matches = has_address_auth_cpucap, }, @@ -2255,22 +2307,35 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_address_auth_metacap, }, { - .desc = "Generic authentication (architected algorithm)", - .capability = ARM64_HAS_GENERIC_AUTH_ARCH, + .desc = "Generic authentication (architected QARMA5 algorithm)", + .capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_GPA_SHIFT, + .field_width = 4, .min_field_value = ID_AA64ISAR1_GPA_ARCHITECTED, .matches = has_cpuid_feature, }, { + .desc = "Generic authentication (architected QARMA3 algorithm)", + .capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .sys_reg = SYS_ID_AA64ISAR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64ISAR2_GPA3_SHIFT, + .field_width = 4, + .min_field_value = ID_AA64ISAR2_GPA3_ARCHITECTED, + .matches = has_cpuid_feature, + }, + { .desc = "Generic authentication (IMP DEF algorithm)", .capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_GPI_SHIFT, + .field_width = 4, .min_field_value = ID_AA64ISAR1_GPI_IMP_DEF, .matches = has_cpuid_feature, }, @@ -2291,6 +2356,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = can_use_gic_priorities, .sys_reg = SYS_ID_AA64PFR0_EL1, .field_pos = ID_AA64PFR0_GIC_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, }, @@ -2302,6 +2368,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, + .field_width = 4, .field_pos = ID_AA64MMFR2_E0PD_SHIFT, .matches = has_cpuid_feature, .min_field_value = 1, @@ -2316,6 +2383,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64ISAR0_EL1, .field_pos = ID_AA64ISAR0_RNDR_SHIFT, + .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, }, @@ -2333,6 +2401,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = bti_enable, .sys_reg = SYS_ID_AA64PFR1_EL1, .field_pos = ID_AA64PFR1_BT_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR1_BT_BTI, .sign = FTR_UNSIGNED, }, @@ -2345,6 +2414,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, .field_pos = ID_AA64PFR1_MTE_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR1_MTE, .sign = FTR_UNSIGNED, .cpu_enable = cpu_enable_mte, @@ -2356,6 +2426,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, .field_pos = ID_AA64PFR1_MTE_SHIFT, + .field_width = 4, .min_field_value = ID_AA64PFR1_MTE_ASYMM, .sign = FTR_UNSIGNED, }, @@ -2367,16 +2438,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_LRCPC_SHIFT, + .field_width = 4, .matches = has_cpuid_feature, .min_field_value = 1, }, {}, }; -#define HWCAP_CPUID_MATCH(reg, field, s, min_value) \ +#define HWCAP_CPUID_MATCH(reg, field, width, s, min_value) \ .matches = has_cpuid_feature, \ .sys_reg = reg, \ .field_pos = field, \ + .field_width = width, \ .sign = s, \ .min_field_value = min_value, @@ -2386,10 +2459,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .hwcap_type = cap_type, \ .hwcap = cap, \ -#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \ +#define HWCAP_CAP(reg, field, width, s, min_value, cap_type, cap) \ { \ __HWCAP_CAP(#cap, cap_type, cap) \ - HWCAP_CPUID_MATCH(reg, field, s, min_value) \ + HWCAP_CPUID_MATCH(reg, field, width, s, min_value) \ } #define HWCAP_MULTI_CAP(list, cap_type, cap) \ @@ -2409,11 +2482,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = { { HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_APA_SHIFT, - FTR_UNSIGNED, ID_AA64ISAR1_APA_ARCHITECTED) + 4, FTR_UNSIGNED, + ID_AA64ISAR1_APA_ARCHITECTED) + }, + { + HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_APA3_SHIFT, + 4, FTR_UNSIGNED, ID_AA64ISAR2_APA3_ARCHITECTED) }, { HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_SHIFT, - FTR_UNSIGNED, ID_AA64ISAR1_API_IMP_DEF) + 4, FTR_UNSIGNED, ID_AA64ISAR1_API_IMP_DEF) }, {}, }; @@ -2421,77 +2499,82 @@ static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = { static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = { { HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPA_SHIFT, - FTR_UNSIGNED, ID_AA64ISAR1_GPA_ARCHITECTED) + 4, FTR_UNSIGNED, ID_AA64ISAR1_GPA_ARCHITECTED) + }, + { + HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_GPA3_SHIFT, + 4, FTR_UNSIGNED, ID_AA64ISAR2_GPA3_ARCHITECTED) }, { HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPI_SHIFT, - FTR_UNSIGNED, ID_AA64ISAR1_GPI_IMP_DEF) + 4, FTR_UNSIGNED, ID_AA64ISAR1_GPI_IMP_DEF) }, {}, }; #endif static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_PMULL), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AES), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA1), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA2), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_SHA512), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_CRC32), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ATOMICS), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA3), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM3), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM4), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FLAGM), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), - HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RNDR_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RNG), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DCPOP), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_DCPODP), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_JSCVT), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FCMA), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_LRCPC), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FRINTTS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FRINT), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SB), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_BF16_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_BF16), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DGH_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DGH), - HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_I8MM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_I8MM), - HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_PMULL), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AES), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA1), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA2), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_SHA512), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_CRC32), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ATOMICS), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA3), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM3), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM4), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FLAGM), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), + HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RNDR_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RNG), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DCPOP), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_DCPODP), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_JSCVT), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FCMA), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_LRCPC), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FRINTTS_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FRINT), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SB), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_BF16_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_BF16), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DGH_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DGH), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_I8MM_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_I8MM), + HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, KERNEL_HWCAP_SVE), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SVEVER_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SVEVER_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_AES, CAP_HWCAP, KERNEL_HWCAP_SVEAES), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_AES_PMULL, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BITPERM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_BITPERM, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BF16_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_BF16, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SHA3_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SHA3, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SM4_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SM4, CAP_HWCAP, KERNEL_HWCAP_SVESM4), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_I8MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_I8MM, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F32MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_F32MM, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM), - HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F64MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_F64MM, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SVEVER_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_SVEVER_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_AES, CAP_HWCAP, KERNEL_HWCAP_SVEAES), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_AES_PMULL, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BITPERM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_BITPERM, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BF16_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_BF16, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SHA3_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_SHA3, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SM4_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_SM4, CAP_HWCAP, KERNEL_HWCAP_SVESM4), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_I8MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_I8MM, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F32MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_F32MM, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F64MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_F64MM, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM), #endif - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS), #ifdef CONFIG_ARM64_BTI - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI), #endif #ifdef CONFIG_ARM64_PTR_AUTH HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA), HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG), #endif #ifdef CONFIG_ARM64_MTE - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE_ASYMM, CAP_HWCAP, KERNEL_HWCAP_MTE3), #endif /* CONFIG_ARM64_MTE */ - HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), - HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), - HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), + HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), + HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), + HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), {}, }; @@ -2520,15 +2603,15 @@ static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope) static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { #ifdef CONFIG_COMPAT HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON), - HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4), + HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4), /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */ - HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP), - HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3), - HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL), - HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES), - HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1), - HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2), - HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32), + HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP), + HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3), + HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL), + HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES), + HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1), + HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2), + HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32), #endif {}, }; diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index 03991eeff643..3006f4324808 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -54,6 +54,9 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu) struct acpi_lpi_state *lpi; struct acpi_processor *pr = per_cpu(processors, cpu); + if (unlikely(!pr || !pr->flags.has_lpi)) + return -EINVAL; + /* * If the PSCI cpu_suspend function hook has not been initialized * idle states must not be enabled, so bail out @@ -61,9 +64,6 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu) if (!psci_ops.cpu_suspend) return -EOPNOTSUPP; - if (unlikely(!pr || !pr->flags.has_lpi)) - return -EINVAL; - count = pr->power.count - 1; if (count <= 0) return -ENODEV; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 591c18a889a5..330b92ea863a 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -97,6 +97,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_ECV] = "ecv", [KERNEL_HWCAP_AFP] = "afp", [KERNEL_HWCAP_RPRES] = "rpres", + [KERNEL_HWCAP_MTE3] = "mte3", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c index 314391a156ee..2b65aae332ce 100644 --- a/arch/arm64/kernel/crash_core.c +++ b/arch/arm64/kernel/crash_core.c @@ -20,6 +20,12 @@ void arch_crash_save_vmcoreinfo(void) { VMCOREINFO_NUMBER(VA_BITS); /* Please note VMCOREINFO_NUMBER() uses "%d", not "%x" */ + vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); + vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); + vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START); + vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); + vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); + vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n", kimage_voffset); vmcoreinfo_append_str("NUMBER(PHYS_OFFSET)=0x%llx\n", diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c new file mode 100644 index 000000000000..3ed39c61a510 --- /dev/null +++ b/arch/arm64/kernel/elfcore.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/coredump.h> +#include <linux/elfcore.h> +#include <linux/kernel.h> +#include <linux/mm.h> + +#include <asm/cpufeature.h> +#include <asm/mte.h> + +#ifndef VMA_ITERATOR +#define VMA_ITERATOR(name, mm, addr) \ + struct mm_struct *name = mm +#define for_each_vma(vmi, vma) \ + for (vma = vmi->mmap; vma; vma = vma->vm_next) +#endif + +#define for_each_mte_vma(vmi, vma) \ + if (system_supports_mte()) \ + for_each_vma(vmi, vma) \ + if (vma->vm_flags & VM_MTE) + +static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma) +{ + if (vma->vm_flags & VM_DONTDUMP) + return 0; + + return vma_pages(vma) * MTE_PAGE_TAG_STORAGE; +} + +/* Derived from dump_user_range(); start/end must be page-aligned */ +static int mte_dump_tag_range(struct coredump_params *cprm, + unsigned long start, unsigned long end) +{ + unsigned long addr; + + for (addr = start; addr < end; addr += PAGE_SIZE) { + char tags[MTE_PAGE_TAG_STORAGE]; + struct page *page = get_dump_page(addr); + + /* + * get_dump_page() returns NULL when encountering an empty + * page table entry that would otherwise have been filled with + * the zero page. Skip the equivalent tag dump which would + * have been all zeros. + */ + if (!page) { + dump_skip(cprm, MTE_PAGE_TAG_STORAGE); + continue; + } + + /* + * Pages mapped in user space as !pte_access_permitted() (e.g. + * PROT_EXEC only) may not have the PG_mte_tagged flag set. + */ + if (!test_bit(PG_mte_tagged, &page->flags)) { + put_page(page); + dump_skip(cprm, MTE_PAGE_TAG_STORAGE); + continue; + } + + mte_save_page_tags(page_address(page), tags); + put_page(page); + if (!dump_emit(cprm, tags, MTE_PAGE_TAG_STORAGE)) + return 0; + } + + return 1; +} + +Elf_Half elf_core_extra_phdrs(void) +{ + struct vm_area_struct *vma; + int vma_count = 0; + VMA_ITERATOR(vmi, current->mm, 0); + + for_each_mte_vma(vmi, vma) + vma_count++; + + return vma_count; +} + +int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset) +{ + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, current->mm, 0); + + for_each_mte_vma(vmi, vma) { + struct elf_phdr phdr; + + phdr.p_type = PT_ARM_MEMTAG_MTE; + phdr.p_offset = offset; + phdr.p_vaddr = vma->vm_start; + phdr.p_paddr = 0; + phdr.p_filesz = mte_vma_tag_dump_size(vma); + phdr.p_memsz = vma->vm_end - vma->vm_start; + offset += phdr.p_filesz; + phdr.p_flags = 0; + phdr.p_align = 0; + + if (!dump_emit(cprm, &phdr, sizeof(phdr))) + return 0; + } + + return 1; +} + +size_t elf_core_extra_data_size(void) +{ + struct vm_area_struct *vma; + size_t data_size = 0; + VMA_ITERATOR(vmi, current->mm, 0); + + for_each_mte_vma(vmi, vma) + data_size += mte_vma_tag_dump_size(vma); + + return data_size; +} + +int elf_core_write_extra_data(struct coredump_params *cprm) +{ + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, current->mm, 0); + + for_each_mte_vma(vmi, vma) { + if (vma->vm_flags & VM_DONTDUMP) + continue; + + if (!mte_dump_tag_range(cprm, vma->vm_start, vma->vm_end)) + return 0; + } + + return 1; +} diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ef7fcefb96bd..878c65aa7206 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -6,6 +6,7 @@ */ #include <linux/context_tracking.h> +#include <linux/kasan.h> #include <linux/linkage.h> #include <linux/lockdep.h> #include <linux/ptrace.h> @@ -56,6 +57,7 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs) { __enter_from_kernel_mode(regs); mte_check_tfsr_entry(); + mte_disable_tco_entry(current); } /* @@ -103,6 +105,7 @@ static __always_inline void __enter_from_user_mode(void) CT_WARN_ON(ct_state() != CONTEXT_USER); user_exit_irqoff(); trace_hardirqs_off_finish(); + mte_disable_tco_entry(current); } static __always_inline void enter_from_user_mode(struct pt_regs *regs) @@ -220,9 +223,26 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) lockdep_hardirqs_on(CALLER_ADDR0); } +#ifdef CONFIG_PREEMPT_DYNAMIC +DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +#define need_irq_preemption() \ + (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) +#else +#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION)) +#endif + static void __sched arm64_preempt_schedule_irq(void) { - lockdep_assert_irqs_disabled(); + if (!need_irq_preemption()) + return; + + /* + * Note: thread_info::preempt_count includes both thread_info::count + * and thread_info::need_resched, and is not equivalent to + * preempt_count(). + */ + if (READ_ONCE(current_thread_info()->preempt_count) != 0) + return; /* * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC @@ -438,14 +458,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - /* - * Note: thread_info::preempt_count includes both thread_info::count - * and thread_info::need_resched, and is not equivalent to - * preempt_count(). - */ - if (IS_ENABLED(CONFIG_PREEMPTION) && - READ_ONCE(current_thread_info()->preempt_count) == 0) - arm64_preempt_schedule_irq(); + arm64_preempt_schedule_irq(); exit_to_kernel_mode(regs); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 772ec2ecf488..ede028dee81b 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -37,18 +37,21 @@ .macro kernel_ventry, el:req, ht:req, regsize:req, label:req .align 7 -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +.Lventry_start\@: .if \el == 0 -alternative_if ARM64_UNMAP_KERNEL_AT_EL0 + /* + * This must be the first instruction of the EL0 vector entries. It is + * skipped by the trampoline vectors, to trigger the cleanup. + */ + b .Lskip_tramp_vectors_cleanup\@ .if \regsize == 64 mrs x30, tpidrro_el0 msr tpidrro_el0, xzr .else mov x30, xzr .endif -alternative_else_nop_endif +.Lskip_tramp_vectors_cleanup\@: .endif -#endif sub sp, sp, #PT_REGS_SIZE #ifdef CONFIG_VMAP_STACK @@ -95,11 +98,15 @@ alternative_else_nop_endif mrs x0, tpidrro_el0 #endif b el\el\ht\()_\regsize\()_\label +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym + .macro tramp_alias, dst, sym, tmp mov_q \dst, TRAMP_VALIAS - add \dst, \dst, #(\sym - .entry.tramp.text) + adr_l \tmp, \sym + add \dst, \dst, \tmp + adr_l \tmp, .entry.tramp.text + sub \dst, \dst, \tmp .endm /* @@ -116,7 +123,7 @@ alternative_cb_end tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 mov w1, #\state -alternative_cb spectre_v4_patch_fw_mitigation_conduit +alternative_cb smccc_patch_fw_mitigation_conduit nop // Patched to SMC/HVC #0 alternative_cb_end .L__asm_ssbd_skip\@: @@ -300,6 +307,7 @@ alternative_else_nop_endif str w21, [sp, #S_SYSCALLNO] .endif +#ifdef CONFIG_ARM64_PSEUDO_NMI /* Save pmr */ alternative_if ARM64_HAS_IRQ_PRIO_MASKING mrs_s x20, SYS_ICC_PMR_EL1 @@ -307,12 +315,6 @@ alternative_if ARM64_HAS_IRQ_PRIO_MASKING mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET msr_s SYS_ICC_PMR_EL1, x20 alternative_else_nop_endif - - /* Re-enable tag checking (TCO set on exception entry) */ -#ifdef CONFIG_ARM64_MTE -alternative_if ARM64_MTE - SET_PSTATE_TCO(0) -alternative_else_nop_endif #endif /* @@ -330,6 +332,7 @@ alternative_else_nop_endif disable_daif .endif +#ifdef CONFIG_ARM64_PSEUDO_NMI /* Restore pmr */ alternative_if ARM64_HAS_IRQ_PRIO_MASKING ldr x20, [sp, #S_PMR_SAVE] @@ -339,6 +342,7 @@ alternative_if ARM64_HAS_IRQ_PRIO_MASKING dsb sy // Ensure priority change is seen by redistributor .L__skip_pmr_sync\@: alternative_else_nop_endif +#endif ldp x21, x22, [sp, #S_PC] // load ELR, SPSR @@ -413,21 +417,26 @@ alternative_else_nop_endif ldp x24, x25, [sp, #16 * 12] ldp x26, x27, [sp, #16 * 13] ldp x28, x29, [sp, #16 * 14] - ldr lr, [sp, #S_LR] - add sp, sp, #PT_REGS_SIZE // restore sp .if \el == 0 -alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 +alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 + ldr lr, [sp, #S_LR] + add sp, sp, #PT_REGS_SIZE // restore sp + eret +alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 bne 4f - msr far_el1, x30 - tramp_alias x30, tramp_exit_native + msr far_el1, x29 + tramp_alias x30, tramp_exit_native, x29 br x30 4: - tramp_alias x30, tramp_exit_compat + tramp_alias x30, tramp_exit_compat, x29 br x30 #endif .else + ldr lr, [sp, #S_LR] + add sp, sp, #PT_REGS_SIZE // restore sp + /* Ensure any device/NC reads complete */ alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412 @@ -594,12 +603,6 @@ SYM_CODE_END(ret_to_user) .popsection // .entry.text -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -/* - * Exception vectors trampoline. - */ - .pushsection ".entry.tramp.text", "ax" - // Move from tramp_pg_dir to swapper_pg_dir .macro tramp_map_kernel, tmp mrs \tmp, ttbr1_el1 @@ -633,12 +636,47 @@ alternative_else_nop_endif */ .endm - .macro tramp_ventry, regsize = 64 + .macro tramp_data_page dst + adr_l \dst, .entry.tramp.text + sub \dst, \dst, PAGE_SIZE + .endm + + .macro tramp_data_read_var dst, var +#ifdef CONFIG_RANDOMIZE_BASE + tramp_data_page \dst + add \dst, \dst, #:lo12:__entry_tramp_data_\var + ldr \dst, [\dst] +#else + ldr \dst, =\var +#endif + .endm + +#define BHB_MITIGATION_NONE 0 +#define BHB_MITIGATION_LOOP 1 +#define BHB_MITIGATION_FW 2 +#define BHB_MITIGATION_INSN 3 + + .macro tramp_ventry, vector_start, regsize, kpti, bhb .align 7 1: .if \regsize == 64 msr tpidrro_el0, x30 // Restored in kernel_ventry .endif + + .if \bhb == BHB_MITIGATION_LOOP + /* + * This sequence must appear before the first indirect branch. i.e. the + * ret out of tramp_ventry. It appears here because x30 is free. + */ + __mitigate_spectre_bhb_loop x30 + .endif // \bhb == BHB_MITIGATION_LOOP + + .if \bhb == BHB_MITIGATION_INSN + clearbhb + isb + .endif // \bhb == BHB_MITIGATION_INSN + + .if \kpti == 1 /* * Defend against branch aliasing attacks by pushing a dummy * entry onto the return stack and using a RET instruction to @@ -648,46 +686,75 @@ alternative_else_nop_endif b . 2: tramp_map_kernel x30 -#ifdef CONFIG_RANDOMIZE_BASE - adr x30, tramp_vectors + PAGE_SIZE alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 - ldr x30, [x30] -#else - ldr x30, =vectors -#endif + tramp_data_read_var x30, vectors alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM - prfm plil1strm, [x30, #(1b - tramp_vectors)] + prfm plil1strm, [x30, #(1b - \vector_start)] alternative_else_nop_endif + msr vbar_el1, x30 - add x30, x30, #(1b - tramp_vectors) isb + .else + ldr x30, =vectors + .endif // \kpti == 1 + + .if \bhb == BHB_MITIGATION_FW + /* + * The firmware sequence must appear before the first indirect branch. + * i.e. the ret out of tramp_ventry. But it also needs the stack to be + * mapped to save/restore the registers the SMC clobbers. + */ + __mitigate_spectre_bhb_fw + .endif // \bhb == BHB_MITIGATION_FW + + add x30, x30, #(1b - \vector_start + 4) ret +.org 1b + 128 // Did we overflow the ventry slot? .endm .macro tramp_exit, regsize = 64 - adr x30, tramp_vectors + tramp_data_read_var x30, this_cpu_vector + get_this_cpu_offset x29 + ldr x30, [x30, x29] + msr vbar_el1, x30 - tramp_unmap_kernel x30 + ldr lr, [sp, #S_LR] + tramp_unmap_kernel x29 .if \regsize == 64 - mrs x30, far_el1 + mrs x29, far_el1 .endif + add sp, sp, #PT_REGS_SIZE // restore sp eret sb .endm - .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) + .macro generate_tramp_vector, kpti, bhb +.Lvector_start\@: .space 0x400 - tramp_ventry - tramp_ventry - tramp_ventry - tramp_ventry + .rept 4 + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb + .endr + .rept 4 + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb + .endr + .endm - tramp_ventry 32 - tramp_ventry 32 - tramp_ventry 32 - tramp_ventry 32 +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +/* + * Exception vectors trampoline. + * The order must match __bp_harden_el1_vectors and the + * arm64_bp_harden_el1_vectors enum. + */ + .pushsection ".entry.tramp.text", "ax" + .align 11 +SYM_CODE_START_NOALIGN(tramp_vectors) +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) @@ -704,13 +771,57 @@ SYM_CODE_END(tramp_exit_compat) .pushsection ".rodata", "a" .align PAGE_SHIFT SYM_DATA_START(__entry_tramp_data_start) +__entry_tramp_data_vectors: .quad vectors +#ifdef CONFIG_ARM_SDE_INTERFACE +__entry_tramp_data___sdei_asm_handler: + .quad __sdei_asm_handler +#endif /* CONFIG_ARM_SDE_INTERFACE */ +__entry_tramp_data_this_cpu_vector: + .quad this_cpu_vector SYM_DATA_END(__entry_tramp_data_start) .popsection // .rodata #endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ /* + * Exception vectors for spectre mitigations on entry from EL1 when + * kpti is not in use. + */ + .macro generate_el1_vector, bhb +.Lvector_start\@: + kernel_ventry 1, t, 64, sync // Synchronous EL1t + kernel_ventry 1, t, 64, irq // IRQ EL1t + kernel_ventry 1, t, 64, fiq // FIQ EL1h + kernel_ventry 1, t, 64, error // Error EL1t + + kernel_ventry 1, h, 64, sync // Synchronous EL1h + kernel_ventry 1, h, 64, irq // IRQ EL1h + kernel_ventry 1, h, 64, fiq // FIQ EL1h + kernel_ventry 1, h, 64, error // Error EL1h + + .rept 4 + tramp_ventry .Lvector_start\@, 64, 0, \bhb + .endr + .rept 4 + tramp_ventry .Lvector_start\@, 32, 0, \bhb + .endr + .endm + +/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ + .pushsection ".entry.text", "ax" + .align 11 +SYM_CODE_START(__bp_harden_el1_vectors) +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + generate_el1_vector bhb=BHB_MITIGATION_LOOP + generate_el1_vector bhb=BHB_MITIGATION_FW + generate_el1_vector bhb=BHB_MITIGATION_INSN +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ +SYM_CODE_END(__bp_harden_el1_vectors) + .popsection + + +/* * Register switch for AArch64. The callee-saved registers need to be saved * and restored. On entry: * x0 = previous task_struct (must be preserved across the switch) @@ -835,14 +946,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) * Remember whether to unmap the kernel on exit. */ 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)] - -#ifdef CONFIG_RANDOMIZE_BASE - adr x4, tramp_vectors + PAGE_SIZE - add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler - ldr x4, [x4] -#else - ldr x4, =__sdei_asm_handler -#endif + tramp_data_read_var x4, __sdei_asm_handler br x4 SYM_CODE_END(__sdei_asm_entry_trampoline) NOKPROBE(__sdei_asm_entry_trampoline) @@ -865,13 +969,6 @@ SYM_CODE_END(__sdei_asm_exit_trampoline) NOKPROBE(__sdei_asm_exit_trampoline) .ltorg .popsection // .entry.tramp.text -#ifdef CONFIG_RANDOMIZE_BASE -.pushsection ".rodata", "a" -SYM_DATA_START(__sdei_asm_trampoline_next_handler) - .quad __sdei_asm_handler -SYM_DATA_END(__sdei_asm_trampoline_next_handler) -.popsection // .rodata -#endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ /* @@ -981,7 +1078,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 br x5 #endif SYM_CODE_END(__sdei_asm_handler) diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index d8e606fe3c21..8a2ceb591686 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -17,7 +17,7 @@ #define FTR_DESC_NAME_LEN 20 #define FTR_DESC_FIELD_LEN 10 #define FTR_ALIAS_NAME_LEN 30 -#define FTR_ALIAS_OPTION_LEN 80 +#define FTR_ALIAS_OPTION_LEN 116 struct ftr_set_desc { char name[FTR_DESC_NAME_LEN]; @@ -71,6 +71,16 @@ static const struct ftr_set_desc isar1 __initconst = { }, }; +static const struct ftr_set_desc isar2 __initconst = { + .name = "id_aa64isar2", + .override = &id_aa64isar2_override, + .fields = { + { "gpa3", ID_AA64ISAR2_GPA3_SHIFT }, + { "apa3", ID_AA64ISAR2_APA3_SHIFT }, + {} + }, +}; + extern struct arm64_ftr_override kaslr_feature_override; static const struct ftr_set_desc kaslr __initconst = { @@ -88,6 +98,7 @@ static const struct ftr_set_desc * const regs[] __initconst = { &mmfr1, &pfr1, &isar1, + &isar2, &kaslr, }; @@ -100,7 +111,8 @@ static const struct { { "arm64.nobti", "id_aa64pfr1.bt=0" }, { "arm64.nopauth", "id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 " - "id_aa64isar1.api=0 id_aa64isar1.apa=0" }, + "id_aa64isar1.api=0 id_aa64isar1.apa=0 " + "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, { "nokaslr", "kaslr.disabled=1" }, }; diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 7eaf1f7c4168..55a1ced8eb77 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -66,6 +66,10 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch); KVM_NVHE_ALIAS(kvm_update_va_mask); KVM_NVHE_ALIAS(kvm_get_kimage_voffset); KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0); +KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter); +KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); +KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); +KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index f418ebc65f95..78b3e0f8e997 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -186,6 +186,11 @@ void mte_check_tfsr_el1(void) } #endif +/* + * This is where we actually resolve the system and process MTE mode + * configuration into an actual value in SCTLR_EL1 that affects + * userspace. + */ static void mte_update_sctlr_user(struct task_struct *task) { /* @@ -199,9 +204,20 @@ static void mte_update_sctlr_user(struct task_struct *task) unsigned long pref, resolved_mte_tcf; pref = __this_cpu_read(mte_tcf_preferred); + /* + * If there is no overlap between the system preferred and + * program requested values go with what was requested. + */ resolved_mte_tcf = (mte_ctrl & pref) ? pref : mte_ctrl; sctlr &= ~SCTLR_EL1_TCF0_MASK; - if (resolved_mte_tcf & MTE_CTRL_TCF_ASYNC) + /* + * Pick an actual setting. The order in which we check for + * set bits and map into register values determines our + * default order. + */ + if (resolved_mte_tcf & MTE_CTRL_TCF_ASYMM) + sctlr |= SCTLR_EL1_TCF0_ASYMM; + else if (resolved_mte_tcf & MTE_CTRL_TCF_ASYNC) sctlr |= SCTLR_EL1_TCF0_ASYNC; else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC) sctlr |= SCTLR_EL1_TCF0_SYNC; @@ -253,6 +269,9 @@ void mte_thread_switch(struct task_struct *next) mte_update_sctlr_user(next); mte_update_gcr_excl(next); + /* TCO may not have been disabled on exception entry for the current task. */ + mte_disable_tco_entry(next); + /* * Check if an async tag exception occurred at EL1. * @@ -293,6 +312,17 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_MTE_TCF_SYNC) mte_ctrl |= MTE_CTRL_TCF_SYNC; + /* + * If the system supports it and both sync and async modes are + * specified then implicitly enable asymmetric mode. + * Userspace could see a mix of both sync and async anyway due + * to differing or changing defaults on CPUs. + */ + if (cpus_have_cap(ARM64_MTE_ASYMM) && + (arg & PR_MTE_TCF_ASYNC) && + (arg & PR_MTE_TCF_SYNC)) + mte_ctrl |= MTE_CTRL_TCF_ASYMM; + task->thread.mte_ctrl = mte_ctrl; if (task == current) { preempt_disable(); @@ -467,6 +497,8 @@ static ssize_t mte_tcf_preferred_show(struct device *dev, return sysfs_emit(buf, "async\n"); case MTE_CTRL_TCF_SYNC: return sysfs_emit(buf, "sync\n"); + case MTE_CTRL_TCF_ASYMM: + return sysfs_emit(buf, "asymm\n"); default: return sysfs_emit(buf, "???\n"); } @@ -482,6 +514,8 @@ static ssize_t mte_tcf_preferred_store(struct device *dev, tcf = MTE_CTRL_TCF_ASYNC; else if (sysfs_streq(buf, "sync")) tcf = MTE_CTRL_TCF_SYNC; + else if (cpus_have_cap(ARM64_MTE_ASYMM) && sysfs_streq(buf, "asymm")) + tcf = MTE_CTRL_TCF_ASYMM; else return -EINVAL; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index cab678ed6618..cb69ff1e6138 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -242,6 +242,16 @@ static struct attribute *armv8_pmuv3_event_attrs[] = { ARMV8_EVENT_ATTR(l2d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L2D_CACHE_LMISS_RD), ARMV8_EVENT_ATTR(l2i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS), ARMV8_EVENT_ATTR(l3d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD), + ARMV8_EVENT_ATTR(trb_wrap, ARMV8_PMUV3_PERFCTR_TRB_WRAP), + ARMV8_EVENT_ATTR(trb_trig, ARMV8_PMUV3_PERFCTR_TRB_TRIG), + ARMV8_EVENT_ATTR(trcextout0, ARMV8_PMUV3_PERFCTR_TRCEXTOUT0), + ARMV8_EVENT_ATTR(trcextout1, ARMV8_PMUV3_PERFCTR_TRCEXTOUT1), + ARMV8_EVENT_ATTR(trcextout2, ARMV8_PMUV3_PERFCTR_TRCEXTOUT2), + ARMV8_EVENT_ATTR(trcextout3, ARMV8_PMUV3_PERFCTR_TRCEXTOUT3), + ARMV8_EVENT_ATTR(cti_trigout4, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT4), + ARMV8_EVENT_ATTR(cti_trigout5, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT5), + ARMV8_EVENT_ATTR(cti_trigout6, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT6), + ARMV8_EVENT_ATTR(cti_trigout7, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT7), ARMV8_EVENT_ATTR(ldst_align_lat, ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT), ARMV8_EVENT_ATTR(ld_align_lat, ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT), ARMV8_EVENT_ATTR(st_align_lat, ARMV8_PMUV3_PERFCTR_ST_ALIGN_LAT), diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 5369e649fa79..7fa97df55e3a 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -635,7 +635,8 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) return -EINVAL; if (system_supports_mte()) - valid_mask |= PR_MTE_TCF_MASK | PR_MTE_TAG_MASK; + valid_mask |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC \ + | PR_MTE_TAG_MASK; if (arg & ~valid_mask) return -EINVAL; diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 902e4084c477..5777929d35bf 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -18,15 +18,18 @@ */ #include <linux/arm-smccc.h> +#include <linux/bpf.h> #include <linux/cpu.h> #include <linux/device.h> #include <linux/nospec.h> #include <linux/prctl.h> #include <linux/sched/task_stack.h> +#include <asm/debug-monitors.h> #include <asm/insn.h> #include <asm/spectre.h> #include <asm/traps.h> +#include <asm/vectors.h> #include <asm/virt.h> /* @@ -96,14 +99,51 @@ static bool spectre_v2_mitigations_off(void) return ret; } +static const char *get_bhb_affected_string(enum mitigation_state bhb_state) +{ + switch (bhb_state) { + case SPECTRE_UNAFFECTED: + return ""; + default: + case SPECTRE_VULNERABLE: + return ", but not BHB"; + case SPECTRE_MITIGATED: + return ", BHB"; + } +} + +static bool _unprivileged_ebpf_enabled(void) +{ +#ifdef CONFIG_BPF_SYSCALL + return !sysctl_unprivileged_bpf_disabled; +#else + return false; +#endif +} + ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) { + enum mitigation_state bhb_state = arm64_get_spectre_bhb_state(); + const char *bhb_str = get_bhb_affected_string(bhb_state); + const char *v2_str = "Branch predictor hardening"; + switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + if (bhb_state == SPECTRE_UNAFFECTED) + return sprintf(buf, "Not affected\n"); + + /* + * Platforms affected by Spectre-BHB can't report + * "Not affected" for Spectre-v2. + */ + v2_str = "CSV2"; + fallthrough; case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Branch predictor hardening\n"); + if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + + return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: @@ -193,17 +233,20 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn) __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT); } -static void call_smc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_smc_arch_workaround_1(void) { arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void call_hvc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_hvc_arch_workaround_1(void) { arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void qcom_link_stack_sanitisation(void) +/* Called during entry so must be noinstr */ +static noinstr void qcom_link_stack_sanitisation(void) { u64 tmp; @@ -554,9 +597,9 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction * to call into firmware to adjust the mitigation state. */ -void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt, - __le32 *origptr, - __le32 *updptr, int nr_inst) +void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) { u32 insn; @@ -770,3 +813,344 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) return -ENODEV; } } + +/* + * Spectre BHB. + * + * A CPU is either: + * - Mitigated by a branchy loop a CPU specific number of times, and listed + * in our "loop mitigated list". + * - Mitigated in software by the firmware Spectre v2 call. + * - Has the ClearBHB instruction to perform the mitigation. + * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no + * software mitigation in the vectors is needed. + * - Has CSV2.3, so is unaffected. + */ +static enum mitigation_state spectre_bhb_state; + +enum mitigation_state arm64_get_spectre_bhb_state(void) +{ + return spectre_bhb_state; +} + +enum bhb_mitigation_bits { + BHB_LOOP, + BHB_FW, + BHB_HW, + BHB_INSN, +}; +static unsigned long system_bhb_mitigations; + +/* + * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any + * SCOPE_SYSTEM call will give the right answer. + */ +u8 spectre_bhb_loop_affected(int scope) +{ + u8 k = 0; + static u8 max_bhb_k; + + if (scope == SCOPE_LOCAL_CPU) { + static const struct midr_range spectre_bhb_k32_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + {}, + }; + static const struct midr_range spectre_bhb_k24_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, + }; + static const struct midr_range spectre_bhb_k8_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), + {}, + }; + + if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) + k = 32; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) + k = 24; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) + k = 8; + + max_bhb_k = max(max_bhb_k, k); + } else { + k = max_bhb_k; + } + + return k; +} + +static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void) +{ + int ret; + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_ARCH_WORKAROUND_3, &res); + + ret = res.a0; + switch (ret) { + case SMCCC_RET_SUCCESS: + return SPECTRE_MITIGATED; + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: + return SPECTRE_UNAFFECTED; + default: + fallthrough; + case SMCCC_RET_NOT_SUPPORTED: + return SPECTRE_VULNERABLE; + } +} + +static bool is_spectre_bhb_fw_affected(int scope) +{ + static bool system_affected; + enum mitigation_state fw_state; + bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE; + static const struct midr_range spectre_bhb_firmware_mitigated_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), + {}, + }; + bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(), + spectre_bhb_firmware_mitigated_list); + + if (scope != SCOPE_LOCAL_CPU) + return system_affected; + + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); + if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) { + system_affected = true; + return true; + } + + return false; +} + +static bool supports_ecbhb(int scope) +{ + u64 mmfr1; + + if (scope == SCOPE_LOCAL_CPU) + mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); + else + mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); + + return cpuid_feature_extract_unsigned_field(mmfr1, + ID_AA64MMFR1_ECBHB_SHIFT); +} + +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, + int scope) +{ + WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); + + if (supports_csv2p3(scope)) + return false; + + if (supports_clearbhb(scope)) + return true; + + if (spectre_bhb_loop_affected(scope)) + return true; + + if (is_spectre_bhb_fw_affected(scope)) + return true; + + return false; +} + +static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) +{ + const char *v = arm64_get_bp_hardening_vector(slot); + + if (slot < 0) + return; + + __this_cpu_write(this_cpu_vector, v); + + /* + * When KPTI is in use, the vectors are switched when exiting to + * user-space. + */ + if (arm64_kernel_unmapped_at_el0()) + return; + + write_sysreg(v, vbar_el1); + isb(); +} + +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) +{ + bp_hardening_cb_t cpu_cb; + enum mitigation_state fw_state, state = SPECTRE_VULNERABLE; + struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); + + if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU)) + return; + + if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) { + /* No point mitigating Spectre-BHB alone. */ + } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { + pr_info_once("spectre-bhb mitigation disabled by compile time option\n"); + } else if (cpu_mitigations_off()) { + pr_info_once("spectre-bhb mitigation disabled by command line option\n"); + } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { + state = SPECTRE_MITIGATED; + set_bit(BHB_HW, &system_bhb_mitigations); + } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) { + /* + * Ensure KVM uses the indirect vector which will have ClearBHB + * added. + */ + if (!data->slot) + data->slot = HYP_VECTOR_INDIRECT; + + this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN); + state = SPECTRE_MITIGATED; + set_bit(BHB_INSN, &system_bhb_mitigations); + } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { + /* + * Ensure KVM uses the indirect vector which will have the + * branchy-loop added. A57/A72-r0 will already have selected + * the spectre-indirect vector, which is sufficient for BHB + * too. + */ + if (!data->slot) + data->slot = HYP_VECTOR_INDIRECT; + + this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP); + state = SPECTRE_MITIGATED; + set_bit(BHB_LOOP, &system_bhb_mitigations); + } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) { + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); + if (fw_state == SPECTRE_MITIGATED) { + /* + * Ensure KVM uses one of the spectre bp_hardening + * vectors. The indirect vector doesn't include the EL3 + * call, so needs upgrading to + * HYP_VECTOR_SPECTRE_INDIRECT. + */ + if (!data->slot || data->slot == HYP_VECTOR_INDIRECT) + data->slot += 1; + + this_cpu_set_vectors(EL1_VECTOR_BHB_FW); + + /* + * The WA3 call in the vectors supersedes the WA1 call + * made during context-switch. Uninstall any firmware + * bp_hardening callback. + */ + cpu_cb = spectre_v2_get_sw_mitigation_cb(); + if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb) + __this_cpu_write(bp_hardening_data.fn, NULL); + + state = SPECTRE_MITIGATED; + set_bit(BHB_FW, &system_bhb_mitigations); + } + } + + update_mitigation_state(&spectre_bhb_state, state); +} + +/* Patched to NOP when enabled */ +void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 1); + + if (test_bit(BHB_LOOP, &system_bhb_mitigations)) + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} + +/* Patched to NOP when enabled */ +void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 1); + + if (test_bit(BHB_FW, &system_bhb_mitigations)) + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} + +/* Patched to correct the immediate */ +void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + u8 rd; + u32 insn; + u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM); + + BUG_ON(nr_inst != 1); /* MOV -> MOV */ + + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) + return; + + insn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); + insn = aarch64_insn_gen_movewide(rd, loop_count, 0, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_ZERO); + *updptr++ = cpu_to_le32(insn); +} + +/* Patched to mov WA3 when supported */ +void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + u8 rd; + u32 insn; + + BUG_ON(nr_inst != 1); /* MOV -> MOV */ + + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || + !test_bit(BHB_FW, &system_bhb_mitigations)) + return; + + insn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); + + insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR, + AARCH64_INSN_VARIANT_32BIT, + AARCH64_INSN_REG_ZR, rd, + ARM_SMCCC_ARCH_WORKAROUND_3); + if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT)) + return; + + *updptr++ = cpu_to_le32(insn); +} + +/* Patched to NOP when not supported */ +void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 2); + + if (test_bit(BHB_INSN, &system_bhb_mitigations)) + return; + + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} + +#ifdef CONFIG_BPF_SYSCALL +#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n" +void unpriv_ebpf_notify(int new_state) +{ + if (spectre_v2_state == SPECTRE_VULNERABLE || + spectre_bhb_state != SPECTRE_MITIGATED) + return; + + if (!new_state) + pr_err("WARNING: %s", EBPF_WARN); +} +#endif diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index f70573928f1b..3505789cf4bd 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -406,9 +406,6 @@ static int __init topology_init(void) { int i; - for_each_online_node(i) - register_one_node(i); - for_each_possible_cpu(i) { struct cpu *cpu = &per_cpu(cpu_data.cpu, i); cpu->hotpluggable = cpu_can_disable(i); diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index d8aaf4b6f432..50fe8eaf7df0 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -11,7 +11,6 @@ #include <linux/errno.h> #include <linux/kernel.h> #include <linux/signal.h> -#include <linux/personality.h> #include <linux/freezer.h> #include <linux/stddef.h> #include <linux/uaccess.h> @@ -577,10 +576,12 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user, { int err; - err = sigframe_alloc(user, &user->fpsimd_offset, - sizeof(struct fpsimd_context)); - if (err) - return err; + if (system_supports_fpsimd()) { + err = sigframe_alloc(user, &user->fpsimd_offset, + sizeof(struct fpsimd_context)); + if (err) + return err; + } /* fault information, if valid */ if (add_all || current->thread.fault_code) { diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index db5159a3055f..12c6864e51e1 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -9,7 +9,6 @@ #include <linux/compat.h> #include <linux/cpufeature.h> -#include <linux/personality.h> #include <linux/sched.h> #include <linux/sched/signal.h> #include <linux/slab.h> diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 70fc42470f13..0529fd57567e 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -9,7 +9,6 @@ #include <linux/bug.h> #include <linux/context_tracking.h> #include <linux/signal.h> -#include <linux/personality.h> #include <linux/kallsyms.h> #include <linux/kprobes.h> #include <linux/spinlock.h> @@ -519,7 +518,7 @@ void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr) NOKPROBE_SYMBOL(do_ptrauth_fault); #define __user_cache_maint(insn, address, res) \ - if (address >= user_addr_max()) { \ + if (address >= TASK_SIZE_MAX) { \ res = -EFAULT; \ } else { \ uaccess_ttbr0_enable(); \ diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 50bab186c49b..edaf0faf766f 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -341,7 +341,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) <= SZ_4K, "Hibernate exit text too big or misaligned") #endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE, +ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE, "Entry trampoline text too big") #endif #ifdef CONFIG_KVM diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index ecc5958e27fe..946e401ef6b0 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1491,10 +1491,7 @@ static int kvm_init_vector_slots(void) base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); - if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) - return 0; - - if (!has_vhe()) { + if (kvm_system_needs_idmapped_vectors() && !has_vhe()) { err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), __BP_HARDEN_HYP_VECS_SZ, &base); if (err) @@ -1870,6 +1867,7 @@ static int kvm_hyp_init_protection(u32 hyp_va_bits) kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1); kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); + kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1); diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index b6b6801d96d5..7839d075729b 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -62,6 +62,10 @@ el1_sync: // Guest trapped into EL2 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ ARM_SMCCC_ARCH_WORKAROUND_2) + cbz w1, wa_epilogue + + eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \ + ARM_SMCCC_ARCH_WORKAROUND_3) cbnz w1, el1_trap wa_epilogue: @@ -192,7 +196,10 @@ SYM_CODE_END(__kvm_hyp_vector) sub sp, sp, #(8 * 4) stp x2, x3, [sp, #(8 * 0)] stp x0, x1, [sp, #(8 * 2)] + alternative_cb spectre_bhb_patch_wa3 + /* Patched to mov WA3 when supported */ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1 + alternative_cb_end smc #0 ldp x2, x3, [sp, #(8 * 0)] add sp, sp, #(8 * 2) @@ -205,6 +212,8 @@ SYM_CODE_END(__kvm_hyp_vector) spectrev2_smccc_wa1_smc .else stp x0, x1, [sp, #-16]! + mitigate_spectre_bhb_loop x0 + mitigate_spectre_bhb_clear_insn .endif .if \indirect != 0 alternative_cb kvm_patch_vector_branch diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 701cfb964905..6379a1e3e6e5 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -174,9 +174,9 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) /* Valid trap. Switch the context: */ if (has_vhe()) { - reg = CPACR_EL1_FPEN; + reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN; if (sve_guest) - reg |= CPACR_EL1_ZEN; + reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; sysreg_clear_set(cpacr_el1, 0, reg); } else { diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h index eea1f6a53723..5ad626527d41 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h +++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h @@ -192,6 +192,11 @@ ARM64_FEATURE_MASK(ID_AA64ISAR1_I8MM) \ ) +#define PVM_ID_AA64ISAR2_ALLOW (\ + ARM64_FEATURE_MASK(ID_AA64ISAR2_GPA3) | \ + ARM64_FEATURE_MASK(ID_AA64ISAR2_APA3) \ + ) + u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id); bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code); diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S index 958734f4d6b0..0c367eb5f4e2 100644 --- a/arch/arm64/kvm/hyp/nvhe/cache.S +++ b/arch/arm64/kvm/hyp/nvhe/cache.S @@ -7,7 +7,8 @@ #include <asm/assembler.h> #include <asm/alternative.h> -SYM_FUNC_START_PI(dcache_clean_inval_poc) +SYM_FUNC_START(__pi_dcache_clean_inval_poc) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(dcache_clean_inval_poc) +SYM_FUNC_END(__pi_dcache_clean_inval_poc) +SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 526a7d6fa86f..cdbe8e246418 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -148,8 +148,10 @@ int hyp_map_vectors(void) phys_addr_t phys; void *bp_base; - if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) + if (!kvm_system_needs_idmapped_vectors()) { + __hyp_bp_vect_base = __bp_harden_hyp_vecs; return 0; + } phys = __hyp_pa(__bp_harden_hyp_vecs); bp_base = (void *)__pkvm_create_private_mapping(phys, diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 792cf6e6ac92..33f5181af330 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -22,6 +22,7 @@ u64 id_aa64pfr0_el1_sys_val; u64 id_aa64pfr1_el1_sys_val; u64 id_aa64isar0_el1_sys_val; u64 id_aa64isar1_el1_sys_val; +u64 id_aa64isar2_el1_sys_val; u64 id_aa64mmfr0_el1_sys_val; u64 id_aa64mmfr1_el1_sys_val; u64 id_aa64mmfr2_el1_sys_val; @@ -183,6 +184,17 @@ static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu) return id_aa64isar1_el1_sys_val & allow_mask; } +static u64 get_pvm_id_aa64isar2(const struct kvm_vcpu *vcpu) +{ + u64 allow_mask = PVM_ID_AA64ISAR2_ALLOW; + + if (!vcpu_has_ptrauth(vcpu)) + allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_APA3) | + ARM64_FEATURE_MASK(ID_AA64ISAR2_GPA3)); + + return id_aa64isar2_el1_sys_val & allow_mask; +} + static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu) { u64 set_mask; @@ -225,6 +237,8 @@ u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id) return get_pvm_id_aa64isar0(vcpu); case SYS_ID_AA64ISAR1_EL1: return get_pvm_id_aa64isar1(vcpu); + case SYS_ID_AA64ISAR2_EL1: + return get_pvm_id_aa64isar2(vcpu); case SYS_ID_AA64MMFR0_EL1: return get_pvm_id_aa64mmfr0(vcpu); case SYS_ID_AA64MMFR1_EL1: diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 11d053fdd604..262dfe03134d 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -10,6 +10,7 @@ #include <linux/kvm_host.h> #include <linux/types.h> #include <linux/jump_label.h> +#include <linux/percpu.h> #include <uapi/linux/psci.h> #include <kvm/arm_psci.h> @@ -24,6 +25,8 @@ #include <asm/fpsimd.h> #include <asm/debug-monitors.h> #include <asm/processor.h> +#include <asm/thread_info.h> +#include <asm/vectors.h> /* VHE specific context */ DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); @@ -38,7 +41,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu) val = read_sysreg(cpacr_el1); val |= CPACR_EL1_TTA; - val &= ~CPACR_EL1_ZEN; + val &= ~(CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN); /* * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to @@ -53,9 +56,9 @@ static void __activate_traps(struct kvm_vcpu *vcpu) if (update_fp_enabled(vcpu)) { if (vcpu_has_sve(vcpu)) - val |= CPACR_EL1_ZEN; + val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; } else { - val &= ~CPACR_EL1_FPEN; + val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN); __activate_traps_fpsimd32(vcpu); } @@ -67,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps); static void __deactivate_traps(struct kvm_vcpu *vcpu) { - extern char vectors[]; /* kernel exception vectors */ + const char *host_vectors = vectors; ___deactivate_traps(vcpu); @@ -81,7 +84,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); - write_sysreg(vectors, vbar_el1); + + if (!arm64_kernel_unmapped_at_el0()) + host_vectors = __this_cpu_read(this_cpu_vector); + write_sysreg(host_vectors, vbar_el1); } NOKPROBE_SYMBOL(__deactivate_traps); diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index 30da78f72b3b..202b8c455724 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -107,6 +107,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) break; } break; + case ARM_SMCCC_ARCH_WORKAROUND_3: + switch (arm64_get_spectre_bhb_state()) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; case ARM_SMCCC_HV_PV_TIME_FEATURES: val[0] = SMCCC_RET_SUCCESS; break; diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 3eae32876897..5918095c90a5 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -46,8 +46,7 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) * specification (ARM DEN 0022A). This means all suspend states * for KVM will preserve the register state. */ - kvm_vcpu_halt(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); + kvm_vcpu_wfi(vcpu); return PSCI_RET_SUCCESS; } @@ -406,7 +405,7 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) { - return 3; /* PSCI version and two workaround registers */ + return 4; /* PSCI version and three workaround registers */ } int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) @@ -420,6 +419,9 @@ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++)) return -EFAULT; + if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++)) + return -EFAULT; + return 0; } @@ -459,6 +461,17 @@ static int get_kernel_wa_level(u64 regid) case SPECTRE_VULNERABLE: return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; } + break; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: + switch (arm64_get_spectre_bhb_state()) { + case SPECTRE_VULNERABLE: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; + case SPECTRE_MITIGATED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL; + case SPECTRE_UNAFFECTED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED; + } + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; } return -EINVAL; @@ -475,6 +488,7 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) break; case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK; break; default: @@ -520,6 +534,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) } case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: if (val & ~KVM_REG_FEATURE_LEVEL_MASK) return -EINVAL; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4dc2fba316ff..baa65292bbc2 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1097,6 +1097,11 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, ARM64_FEATURE_MASK(ID_AA64ISAR1_GPA) | ARM64_FEATURE_MASK(ID_AA64ISAR1_GPI)); break; + case SYS_ID_AA64ISAR2_EL1: + if (!vcpu_has_ptrauth(vcpu)) + val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_APA3) | + ARM64_FEATURE_MASK(ID_AA64ISAR2_GPA3)); + break; case SYS_ID_AA64DFR0_EL1: /* Limit debug to ARMv8.0 */ val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER); diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index 7068da080799..49837d3a3ef5 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -248,6 +248,8 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu, IRQCHIP_STATE_PENDING, &val); WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + } else if (vgic_irq_is_mapped_level(irq)) { + val = vgic_get_phys_line_level(irq); } else { val = irq_is_pending(irq); } diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S index 1fd5d790ab80..ebde40e7fa2b 100644 --- a/arch/arm64/lib/clear_page.S +++ b/arch/arm64/lib/clear_page.S @@ -14,7 +14,7 @@ * Parameters: * x0 - dest */ -SYM_FUNC_START_PI(clear_page) +SYM_FUNC_START(__pi_clear_page) mrs x1, dczid_el0 tbnz x1, #4, 2f /* Branch if DC ZVA is prohibited */ and w1, w1, #0xf @@ -35,5 +35,6 @@ SYM_FUNC_START_PI(clear_page) tst x0, #(PAGE_SIZE - 1) b.ne 2b ret -SYM_FUNC_END_PI(clear_page) +SYM_FUNC_END(__pi_clear_page) +SYM_FUNC_ALIAS(clear_page, __pi_clear_page) EXPORT_SYMBOL(clear_page) diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index 29144f4cd449..c336d2ffdec5 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -17,7 +17,7 @@ * x0 - dest * x1 - src */ -SYM_FUNC_START_PI(copy_page) +SYM_FUNC_START(__pi_copy_page) alternative_if ARM64_HAS_NO_HW_PREFETCH // Prefetch three cache lines ahead. prfm pldl1strm, [x1, #128] @@ -75,5 +75,6 @@ alternative_else_nop_endif stnp x16, x17, [x0, #112 - 256] ret -SYM_FUNC_END_PI(copy_page) +SYM_FUNC_END(__pi_copy_page) +SYM_FUNC_ALIAS(copy_page, __pi_copy_page) EXPORT_SYMBOL(copy_page) diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S index 0f9e10ecda23..8340dccff46f 100644 --- a/arch/arm64/lib/crc32.S +++ b/arch/arm64/lib/crc32.S @@ -11,7 +11,44 @@ .arch armv8-a+crc - .macro __crc32, c + .macro byteorder, reg, be + .if \be +CPU_LE( rev \reg, \reg ) + .else +CPU_BE( rev \reg, \reg ) + .endif + .endm + + .macro byteorder16, reg, be + .if \be +CPU_LE( rev16 \reg, \reg ) + .else +CPU_BE( rev16 \reg, \reg ) + .endif + .endm + + .macro bitorder, reg, be + .if \be + rbit \reg, \reg + .endif + .endm + + .macro bitorder16, reg, be + .if \be + rbit \reg, \reg + lsr \reg, \reg, #16 + .endif + .endm + + .macro bitorder8, reg, be + .if \be + rbit \reg, \reg + lsr \reg, \reg, #24 + .endif + .endm + + .macro __crc32, c, be=0 + bitorder w0, \be cmp x2, #16 b.lt 8f // less than 16 bytes @@ -24,10 +61,14 @@ add x8, x8, x1 add x1, x1, x7 ldp x5, x6, [x8] -CPU_BE( rev x3, x3 ) -CPU_BE( rev x4, x4 ) -CPU_BE( rev x5, x5 ) -CPU_BE( rev x6, x6 ) + byteorder x3, \be + byteorder x4, \be + byteorder x5, \be + byteorder x6, \be + bitorder x3, \be + bitorder x4, \be + bitorder x5, \be + bitorder x6, \be tst x7, #8 crc32\c\()x w8, w0, x3 @@ -55,33 +96,43 @@ CPU_BE( rev x6, x6 ) 32: ldp x3, x4, [x1], #32 sub x2, x2, #32 ldp x5, x6, [x1, #-16] -CPU_BE( rev x3, x3 ) -CPU_BE( rev x4, x4 ) -CPU_BE( rev x5, x5 ) -CPU_BE( rev x6, x6 ) + byteorder x3, \be + byteorder x4, \be + byteorder x5, \be + byteorder x6, \be + bitorder x3, \be + bitorder x4, \be + bitorder x5, \be + bitorder x6, \be crc32\c\()x w0, w0, x3 crc32\c\()x w0, w0, x4 crc32\c\()x w0, w0, x5 crc32\c\()x w0, w0, x6 cbnz x2, 32b -0: ret +0: bitorder w0, \be + ret 8: tbz x2, #3, 4f ldr x3, [x1], #8 -CPU_BE( rev x3, x3 ) + byteorder x3, \be + bitorder x3, \be crc32\c\()x w0, w0, x3 4: tbz x2, #2, 2f ldr w3, [x1], #4 -CPU_BE( rev w3, w3 ) + byteorder w3, \be + bitorder w3, \be crc32\c\()w w0, w0, w3 2: tbz x2, #1, 1f ldrh w3, [x1], #2 -CPU_BE( rev16 w3, w3 ) + byteorder16 w3, \be + bitorder16 w3, \be crc32\c\()h w0, w0, w3 1: tbz x2, #0, 0f ldrb w3, [x1] + bitorder8 w3, \be crc32\c\()b w0, w0, w3 -0: ret +0: bitorder w0, \be + ret .endm .align 5 @@ -99,3 +150,11 @@ alternative_if_not ARM64_HAS_CRC32 alternative_else_nop_endif __crc32 c SYM_FUNC_END(__crc32c_le) + + .align 5 +SYM_FUNC_START(crc32_be) +alternative_if_not ARM64_HAS_CRC32 + b crc32_be_base +alternative_else_nop_endif + __crc32 be=1 +SYM_FUNC_END(crc32_be) diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index fccfe363e567..5e90887deec4 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -578,10 +578,16 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg, switch (type) { case AARCH64_INSN_LDST_LOAD_EX: + case AARCH64_INSN_LDST_LOAD_ACQ_EX: insn = aarch64_insn_get_load_ex_value(); + if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX) + insn |= BIT(15); break; case AARCH64_INSN_LDST_STORE_EX: + case AARCH64_INSN_LDST_STORE_REL_EX: insn = aarch64_insn_get_store_ex_value(); + if (type == AARCH64_INSN_LDST_STORE_REL_EX) + insn |= BIT(15); break; default: pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type); @@ -603,12 +609,65 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg, state); } -u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result, - enum aarch64_insn_register address, - enum aarch64_insn_register value, - enum aarch64_insn_size_type size) +#ifdef CONFIG_ARM64_LSE_ATOMICS +static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type, + u32 insn) { - u32 insn = aarch64_insn_get_ldadd_value(); + u32 order; + + switch (type) { + case AARCH64_INSN_MEM_ORDER_NONE: + order = 0; + break; + case AARCH64_INSN_MEM_ORDER_ACQ: + order = 2; + break; + case AARCH64_INSN_MEM_ORDER_REL: + order = 1; + break; + case AARCH64_INSN_MEM_ORDER_ACQREL: + order = 3; + break; + default: + pr_err("%s: unknown mem order %d\n", __func__, type); + return AARCH64_BREAK_FAULT; + } + + insn &= ~GENMASK(23, 22); + insn |= order << 22; + + return insn; +} + +u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_atomic_op op, + enum aarch64_insn_mem_order_type order) +{ + u32 insn; + + switch (op) { + case AARCH64_INSN_MEM_ATOMIC_ADD: + insn = aarch64_insn_get_ldadd_value(); + break; + case AARCH64_INSN_MEM_ATOMIC_CLR: + insn = aarch64_insn_get_ldclr_value(); + break; + case AARCH64_INSN_MEM_ATOMIC_EOR: + insn = aarch64_insn_get_ldeor_value(); + break; + case AARCH64_INSN_MEM_ATOMIC_SET: + insn = aarch64_insn_get_ldset_value(); + break; + case AARCH64_INSN_MEM_ATOMIC_SWP: + insn = aarch64_insn_get_swp_value(); + break; + default: + pr_err("%s: unimplemented mem atomic op %d\n", __func__, op); + return AARCH64_BREAK_FAULT; + } switch (size) { case AARCH64_INSN_SIZE_32: @@ -621,6 +680,8 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result, insn = aarch64_insn_encode_ldst_size(size, insn); + insn = aarch64_insn_encode_ldst_order(order, insn); + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, result); @@ -631,17 +692,68 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result, value); } -u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address, - enum aarch64_insn_register value, - enum aarch64_insn_size_type size) +static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type, + u32 insn) { - /* - * STADD is simply encoded as an alias for LDADD with XZR as - * the destination register. - */ - return aarch64_insn_gen_ldadd(AARCH64_INSN_REG_ZR, address, - value, size); + u32 order; + + switch (type) { + case AARCH64_INSN_MEM_ORDER_NONE: + order = 0; + break; + case AARCH64_INSN_MEM_ORDER_ACQ: + order = BIT(22); + break; + case AARCH64_INSN_MEM_ORDER_REL: + order = BIT(15); + break; + case AARCH64_INSN_MEM_ORDER_ACQREL: + order = BIT(15) | BIT(22); + break; + default: + pr_err("%s: unknown mem order %d\n", __func__, type); + return AARCH64_BREAK_FAULT; + } + + insn &= ~(BIT(15) | BIT(22)); + insn |= order; + + return insn; +} + +u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, + enum aarch64_insn_register address, + enum aarch64_insn_register value, + enum aarch64_insn_size_type size, + enum aarch64_insn_mem_order_type order) +{ + u32 insn; + + switch (size) { + case AARCH64_INSN_SIZE_32: + case AARCH64_INSN_SIZE_64: + break; + default: + pr_err("%s: unimplemented size encoding %d\n", __func__, size); + return AARCH64_BREAK_FAULT; + } + + insn = aarch64_insn_get_cas_value(); + + insn = aarch64_insn_encode_ldst_size(size, insn); + + insn = aarch64_insn_encode_cas_order(order, insn); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, + result); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, + address); + + return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, + value); } +#endif static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type, enum aarch64_insn_prfm_target target, @@ -1379,7 +1491,7 @@ static u32 aarch64_encode_immediate(u64 imm, * Compute the rotation to get a continuous set of * ones, with the first bit set at position 0 */ - ror = fls(~imm); + ror = fls64(~imm); } /* @@ -1456,3 +1568,48 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn); return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm); } + +u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) +{ + u32 opt; + u32 insn; + + switch (type) { + case AARCH64_INSN_MB_SY: + opt = 0xf; + break; + case AARCH64_INSN_MB_ST: + opt = 0xe; + break; + case AARCH64_INSN_MB_LD: + opt = 0xd; + break; + case AARCH64_INSN_MB_ISH: + opt = 0xb; + break; + case AARCH64_INSN_MB_ISHST: + opt = 0xa; + break; + case AARCH64_INSN_MB_ISHLD: + opt = 0x9; + break; + case AARCH64_INSN_MB_NSH: + opt = 0x7; + break; + case AARCH64_INSN_MB_NSHST: + opt = 0x6; + break; + case AARCH64_INSN_MB_NSHLD: + opt = 0x5; + break; + default: + pr_err("%s: unknown dmb type %d\n", __func__, type); + return AARCH64_BREAK_FAULT; + } + + insn = aarch64_insn_get_dmb_value(); + insn &= ~GENMASK(11, 8); + insn |= (opt << 8); + + return insn; +} diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S index 7c2276fdab54..37a9f2a4f7f4 100644 --- a/arch/arm64/lib/memchr.S +++ b/arch/arm64/lib/memchr.S @@ -38,7 +38,7 @@ .p2align 4 nop -SYM_FUNC_START_WEAK_PI(memchr) +SYM_FUNC_START(__pi_memchr) and chrin, chrin, #0xff lsr wordcnt, cntin, #3 cbz wordcnt, L(byte_loop) @@ -71,5 +71,6 @@ CPU_LE( rev tmp, tmp) L(not_found): mov result, #0 ret -SYM_FUNC_END_PI(memchr) +SYM_FUNC_END(__pi_memchr) +SYM_FUNC_ALIAS_WEAK(memchr, __pi_memchr) EXPORT_SYMBOL_NOKASAN(memchr) diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S index 7d956384222f..a5ccf2c55f91 100644 --- a/arch/arm64/lib/memcmp.S +++ b/arch/arm64/lib/memcmp.S @@ -32,7 +32,7 @@ #define tmp1 x7 #define tmp2 x8 -SYM_FUNC_START_WEAK_PI(memcmp) +SYM_FUNC_START(__pi_memcmp) subs limit, limit, 8 b.lo L(less8) @@ -134,6 +134,6 @@ L(byte_loop): b.eq L(byte_loop) sub result, data1w, data2w ret - -SYM_FUNC_END_PI(memcmp) +SYM_FUNC_END(__pi_memcmp) +SYM_FUNC_ALIAS_WEAK(memcmp, __pi_memcmp) EXPORT_SYMBOL_NOKASAN(memcmp) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index b82fd64ee1e1..4ab48d49c451 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -57,10 +57,7 @@ The loop tail is handled by always copying 64 bytes from the end. */ -SYM_FUNC_START_ALIAS(__memmove) -SYM_FUNC_START_WEAK_ALIAS_PI(memmove) -SYM_FUNC_START_ALIAS(__memcpy) -SYM_FUNC_START_WEAK_PI(memcpy) +SYM_FUNC_START(__pi_memcpy) add srcend, src, count add dstend, dstin, count cmp count, 128 @@ -241,12 +238,16 @@ L(copy64_from_start): stp B_l, B_h, [dstin, 16] stp C_l, C_h, [dstin] ret +SYM_FUNC_END(__pi_memcpy) -SYM_FUNC_END_PI(memcpy) -EXPORT_SYMBOL(memcpy) -SYM_FUNC_END_ALIAS(__memcpy) +SYM_FUNC_ALIAS(__memcpy, __pi_memcpy) EXPORT_SYMBOL(__memcpy) -SYM_FUNC_END_ALIAS_PI(memmove) -EXPORT_SYMBOL(memmove) -SYM_FUNC_END_ALIAS(__memmove) +SYM_FUNC_ALIAS_WEAK(memcpy, __memcpy) +EXPORT_SYMBOL(memcpy) + +SYM_FUNC_ALIAS(__pi_memmove, __pi_memcpy) + +SYM_FUNC_ALIAS(__memmove, __pi_memmove) EXPORT_SYMBOL(__memmove) +SYM_FUNC_ALIAS_WEAK(memmove, __memmove) +EXPORT_SYMBOL(memmove) diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S index a9c1c9a01ea9..a5aebe82ad73 100644 --- a/arch/arm64/lib/memset.S +++ b/arch/arm64/lib/memset.S @@ -42,8 +42,7 @@ dst .req x8 tmp3w .req w9 tmp3 .req x9 -SYM_FUNC_START_ALIAS(__memset) -SYM_FUNC_START_WEAK_PI(memset) +SYM_FUNC_START(__pi_memset) mov dst, dstin /* Preserve return value. */ and A_lw, val, #255 orr A_lw, A_lw, A_lw, lsl #8 @@ -202,7 +201,10 @@ SYM_FUNC_START_WEAK_PI(memset) ands count, count, zva_bits_x b.ne .Ltail_maybe_long ret -SYM_FUNC_END_PI(memset) -EXPORT_SYMBOL(memset) -SYM_FUNC_END_ALIAS(__memset) +SYM_FUNC_END(__pi_memset) + +SYM_FUNC_ALIAS(__memset, __pi_memset) EXPORT_SYMBOL(__memset) + +SYM_FUNC_ALIAS_WEAK(memset, __pi_memset) +EXPORT_SYMBOL(memset) diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index f531dcb95174..8590af3c98c0 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -134,7 +134,7 @@ SYM_FUNC_END(mte_copy_tags_to_user) /* * Save the tags in a page * x0 - page address - * x1 - tag storage + * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes */ SYM_FUNC_START(mte_save_page_tags) multitag_transfer_size x7, x5 @@ -158,7 +158,7 @@ SYM_FUNC_END(mte_save_page_tags) /* * Restore the tags in a page * x0 - page address - * x1 - tag storage + * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes */ SYM_FUNC_START(mte_restore_page_tags) multitag_transfer_size x7, x5 diff --git a/arch/arm64/lib/strchr.S b/arch/arm64/lib/strchr.S index 1f47eae3b0d6..94ee67a6b212 100644 --- a/arch/arm64/lib/strchr.S +++ b/arch/arm64/lib/strchr.S @@ -18,7 +18,7 @@ * Returns: * x0 - address of first occurrence of 'c' or 0 */ -SYM_FUNC_START_WEAK(strchr) +SYM_FUNC_START(__pi_strchr) and w1, w1, #0xff 1: ldrb w2, [x0], #1 cmp w2, w1 @@ -28,5 +28,7 @@ SYM_FUNC_START_WEAK(strchr) cmp w2, w1 csel x0, x0, xzr, eq ret -SYM_FUNC_END(strchr) +SYM_FUNC_END(__pi_strchr) + +SYM_FUNC_ALIAS_WEAK(strchr, __pi_strchr) EXPORT_SYMBOL_NOKASAN(strchr) diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S index 83bcad72ec97..9b89b4533607 100644 --- a/arch/arm64/lib/strcmp.S +++ b/arch/arm64/lib/strcmp.S @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2012-2021, Arm Limited. + * Copyright (c) 2012-2022, Arm Limited. * * Adapted from the original at: - * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/strcmp.S + * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S */ #include <linux/linkage.h> @@ -11,166 +11,180 @@ /* Assumptions: * - * ARMv8-a, AArch64 + * ARMv8-a, AArch64. + * MTE compatible. */ #define L(label) .L ## label #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f -#define REP8_80 0x8080808080808080 -/* Parameters and result. */ #define src1 x0 #define src2 x1 #define result x0 -/* Internal variables. */ #define data1 x2 #define data1w w2 #define data2 x3 #define data2w w3 #define has_nul x4 #define diff x5 +#define off1 x5 #define syndrome x6 -#define tmp1 x7 -#define tmp2 x8 -#define tmp3 x9 -#define zeroones x10 -#define pos x11 - - /* Start of performance-critical section -- one 64B cache line. */ - .align 6 -SYM_FUNC_START_WEAK_PI(strcmp) - eor tmp1, src1, src2 - mov zeroones, #REP8_01 - tst tmp1, #7 +#define tmp x6 +#define data3 x7 +#define zeroones x8 +#define shift x9 +#define off2 x10 + +/* On big-endian early bytes are at MSB and on little-endian LSB. + LS_FW means shifting towards early bytes. */ +#ifdef __AARCH64EB__ +# define LS_FW lsl +#else +# define LS_FW lsr +#endif + +/* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. + Since carry propagation makes 0x1 bytes before a NUL byte appear + NUL too in big-endian, byte-reverse the data before the NUL check. */ + + +SYM_FUNC_START(__pi_strcmp) + sub off2, src2, src1 + mov zeroones, REP8_01 + and tmp, src1, 7 + tst off2, 7 b.ne L(misaligned8) - ands tmp1, src1, #7 - b.ne L(mutual_align) - /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 - (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - can be done in parallel across the entire word. */ + cbnz tmp, L(mutual_align) + + .p2align 4 + L(loop_aligned): - ldr data1, [src1], #8 - ldr data2, [src2], #8 + ldr data2, [src1, off2] + ldr data1, [src1], 8 L(start_realigned): - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ +#ifdef __AARCH64EB__ + rev tmp, data1 + sub has_nul, tmp, zeroones + orr tmp, tmp, REP8_7f +#else + sub has_nul, data1, zeroones + orr tmp, data1, REP8_7f +#endif + bics has_nul, has_nul, tmp /* Non-zero if NUL terminator. */ + ccmp data1, data2, 0, eq + b.eq L(loop_aligned) +#ifdef __AARCH64EB__ + rev has_nul, has_nul +#endif + eor diff, data1, data2 orr syndrome, diff, has_nul - cbz syndrome, L(loop_aligned) - /* End of performance-critical section -- one 64B cache line. */ - L(end): -#ifndef __AARCH64EB__ +#ifndef __AARCH64EB__ rev syndrome, syndrome rev data1, data1 - /* The MS-non-zero bit of the syndrome marks either the first bit - that is different, or the top bit of the first zero byte. - Shifting left now will bring the critical information into the - top bits. */ - clz pos, syndrome rev data2, data2 - lsl data1, data1, pos - lsl data2, data2, pos - /* But we need to zero-extend (char is unsigned) the value and then - perform a signed 32-bit subtraction. */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 - ret -#else - /* For big-endian we cannot use the trick with the syndrome value - as carry-propagation can corrupt the upper bits if the trailing - bytes in the string contain 0x01. */ - /* However, if there is no NUL byte in the dword, we can generate - the result directly. We can't just subtract the bytes as the - MSB might be significant. */ - cbnz has_nul, 1f - cmp data1, data2 - cset result, ne - cneg result, result, lo - ret -1: - /* Re-compute the NUL-byte detection, using a byte-reversed value. */ - rev tmp3, data1 - sub tmp1, tmp3, zeroones - orr tmp2, tmp3, #REP8_7f - bic has_nul, tmp1, tmp2 - rev has_nul, has_nul - orr syndrome, diff, has_nul - clz pos, syndrome - /* The MS-non-zero bit of the syndrome marks either the first bit - that is different, or the top bit of the first zero byte. +#endif + clz shift, syndrome + /* The most-significant-non-zero bit of the syndrome marks either the + first bit that is different, or the top bit of the first zero byte. Shifting left now will bring the critical information into the top bits. */ - lsl data1, data1, pos - lsl data2, data2, pos + lsl data1, data1, shift + lsl data2, data2, shift /* But we need to zero-extend (char is unsigned) the value and then perform a signed 32-bit subtraction. */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 + lsr data1, data1, 56 + sub result, data1, data2, lsr 56 ret -#endif + + .p2align 4 L(mutual_align): /* Sources are mutually aligned, but are not currently at an alignment boundary. Round down the addresses and then mask off - the bytes that preceed the start point. */ - bic src1, src1, #7 - bic src2, src2, #7 - lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ - ldr data1, [src1], #8 - neg tmp1, tmp1 /* Bits to alignment -64. */ - ldr data2, [src2], #8 - mov tmp2, #~0 -#ifdef __AARCH64EB__ - /* Big-endian. Early bytes are at MSB. */ - lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ -#else - /* Little-endian. Early bytes are at LSB. */ - lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ -#endif - orr data1, data1, tmp2 - orr data2, data2, tmp2 + the bytes that precede the start point. */ + bic src1, src1, 7 + ldr data2, [src1, off2] + ldr data1, [src1], 8 + neg shift, src2, lsl 3 /* Bits to alignment -64. */ + mov tmp, -1 + LS_FW tmp, tmp, shift + orr data1, data1, tmp + orr data2, data2, tmp b L(start_realigned) L(misaligned8): /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always - checking to make sure that we don't access beyond page boundary in - SRC2. */ - tst src1, #7 - b.eq L(loop_misaligned) + checking to make sure that we don't access beyond the end of SRC2. */ + cbz tmp, L(src1_aligned) L(do_misaligned): - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - cmp data1w, #1 - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + ldrb data1w, [src1], 1 + ldrb data2w, [src2], 1 + cmp data1w, 0 + ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ b.ne L(done) - tst src1, #7 + tst src1, 7 b.ne L(do_misaligned) -L(loop_misaligned): - /* Test if we are within the last dword of the end of a 4K page. If - yes then jump back to the misaligned loop to copy a byte at a time. */ - and tmp1, src2, #0xff8 - eor tmp1, tmp1, #0xff8 - cbz tmp1, L(do_misaligned) - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ +L(src1_aligned): + neg shift, src2, lsl 3 + bic src2, src2, 7 + ldr data3, [src2], 8 +#ifdef __AARCH64EB__ + rev data3, data3 +#endif + lsr tmp, zeroones, shift + orr data3, data3, tmp + sub has_nul, data3, zeroones + orr tmp, data3, REP8_7f + bics has_nul, has_nul, tmp + b.ne L(tail) + + sub off1, src2, src1 + + .p2align 4 + +L(loop_unaligned): + ldr data3, [src1, off1] + ldr data2, [src1, off2] +#ifdef __AARCH64EB__ + rev data3, data3 +#endif + sub has_nul, data3, zeroones + orr tmp, data3, REP8_7f + ldr data1, [src1], 8 + bics has_nul, has_nul, tmp + ccmp data1, data2, 0, eq + b.eq L(loop_unaligned) + + lsl tmp, has_nul, shift +#ifdef __AARCH64EB__ + rev tmp, tmp +#endif + eor diff, data1, data2 + orr syndrome, diff, tmp + cbnz syndrome, L(end) +L(tail): + ldr data1, [src1] + neg shift, shift + lsr data2, data3, shift + lsr has_nul, has_nul, shift +#ifdef __AARCH64EB__ + rev data2, data2 + rev has_nul, has_nul +#endif + eor diff, data1, data2 orr syndrome, diff, has_nul - cbz syndrome, L(loop_misaligned) b L(end) L(done): sub result, data1, data2 ret - -SYM_FUNC_END_PI(strcmp) -EXPORT_SYMBOL_NOHWKASAN(strcmp) +SYM_FUNC_END(__pi_strcmp) +SYM_FUNC_ALIAS_WEAK(strcmp, __pi_strcmp) +EXPORT_SYMBOL_NOKASAN(strcmp) diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S index 1648790e91b3..4919fe81ae54 100644 --- a/arch/arm64/lib/strlen.S +++ b/arch/arm64/lib/strlen.S @@ -79,7 +79,7 @@ whether the first fetch, which may be misaligned, crosses a page boundary. */ -SYM_FUNC_START_WEAK_PI(strlen) +SYM_FUNC_START(__pi_strlen) and tmp1, srcin, MIN_PAGE_SIZE - 1 mov zeroones, REP8_01 cmp tmp1, MIN_PAGE_SIZE - 16 @@ -208,6 +208,6 @@ L(page_cross): csel data1, data1, tmp4, eq csel data2, data2, tmp2, eq b L(page_cross_entry) - -SYM_FUNC_END_PI(strlen) +SYM_FUNC_END(__pi_strlen) +SYM_FUNC_ALIAS_WEAK(strlen, __pi_strlen) EXPORT_SYMBOL_NOKASAN(strlen) diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S index e42bcfcd37e6..fe7bbc0b42a7 100644 --- a/arch/arm64/lib/strncmp.S +++ b/arch/arm64/lib/strncmp.S @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2013-2021, Arm Limited. + * Copyright (c) 2013-2022, Arm Limited. * * Adapted from the original at: - * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/strncmp.S + * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strncmp.S */ #include <linux/linkage.h> @@ -11,14 +11,14 @@ /* Assumptions: * - * ARMv8-a, AArch64 + * ARMv8-a, AArch64. + * MTE compatible. */ #define L(label) .L ## label #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f -#define REP8_80 0x8080808080808080 /* Parameters and result. */ #define src1 x0 @@ -39,12 +39,26 @@ #define tmp3 x10 #define zeroones x11 #define pos x12 -#define limit_wd x13 -#define mask x14 -#define endloop x15 +#define mask x13 +#define endloop x14 #define count mask +#define offset pos +#define neg_offset x15 -SYM_FUNC_START_WEAK_PI(strncmp) +/* Define endian dependent shift operations. + On big-endian early bytes are at MSB and on little-endian LSB. + LS_FW means shifting towards early bytes. + LS_BK means shifting towards later bytes. + */ +#ifdef __AARCH64EB__ +#define LS_FW lsl +#define LS_BK lsr +#else +#define LS_FW lsr +#define LS_BK lsl +#endif + +SYM_FUNC_START(__pi_strncmp) cbz limit, L(ret0) eor tmp1, src1, src2 mov zeroones, #REP8_01 @@ -52,9 +66,6 @@ SYM_FUNC_START_WEAK_PI(strncmp) and count, src1, #7 b.ne L(misaligned8) cbnz count, L(mutual_align) - /* Calculate the number of full and partial words -1. */ - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and @@ -64,56 +75,52 @@ L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 L(start_realigned): - subs limit_wd, limit_wd, #1 + subs limit, limit, #8 sub tmp1, data1, zeroones orr tmp2, data1, #REP8_7f eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, pl /* Last Dword or differences. */ + csinv endloop, diff, xzr, hi /* Last Dword or differences. */ bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ ccmp endloop, #0, #0, eq b.eq L(loop_aligned) /* End of main loop */ - /* Not reached the limit, must have found the end or a diff. */ - tbz limit_wd, #63, L(not_limit) - - /* Limit % 8 == 0 => all bytes significant. */ - ands limit, limit, #7 - b.eq L(not_limit) - - lsl limit, limit, #3 /* Bits -> bytes. */ - mov mask, #~0 -#ifdef __AARCH64EB__ - lsr mask, mask, limit -#else - lsl mask, mask, limit -#endif - bic data1, data1, mask - bic data2, data2, mask - - /* Make sure that the NUL byte is marked in the syndrome. */ - orr has_nul, has_nul, mask - -L(not_limit): +L(full_check): +#ifndef __AARCH64EB__ orr syndrome, diff, has_nul - -#ifndef __AARCH64EB__ + add limit, limit, 8 /* Rewind limit to before last subs. */ +L(syndrome_check): + /* Limit was reached. Check if the NUL byte or the difference + is before the limit. */ rev syndrome, syndrome rev data1, data1 - /* The MS-non-zero bit of the syndrome marks either the first bit - that is different, or the top bit of the first zero byte. - Shifting left now will bring the critical information into the - top bits. */ clz pos, syndrome rev data2, data2 lsl data1, data1, pos + cmp limit, pos, lsr #3 lsl data2, data2, pos /* But we need to zero-extend (char is unsigned) the value and then perform a signed 32-bit subtraction. */ lsr data1, data1, #56 sub result, data1, data2, lsr #56 + csel result, result, xzr, hi ret #else + /* Not reached the limit, must have found the end or a diff. */ + tbz limit, #63, L(not_limit) + add tmp1, limit, 8 + cbz limit, L(not_limit) + + lsl limit, tmp1, #3 /* Bits -> bytes. */ + mov mask, #~0 + lsr mask, mask, limit + bic data1, data1, mask + bic data2, data2, mask + + /* Make sure that the NUL byte is marked in the syndrome. */ + orr has_nul, has_nul, mask + +L(not_limit): /* For big-endian we cannot use the trick with the syndrome value as carry-propagation can corrupt the upper bits if the trailing bytes in the string contain 0x01. */ @@ -134,10 +141,11 @@ L(not_limit): rev has_nul, has_nul orr syndrome, diff, has_nul clz pos, syndrome - /* The MS-non-zero bit of the syndrome marks either the first bit - that is different, or the top bit of the first zero byte. + /* The most-significant-non-zero bit of the syndrome marks either the + first bit that is different, or the top bit of the first zero byte. Shifting left now will bring the critical information into the top bits. */ +L(end_quick): lsl data1, data1, pos lsl data2, data2, pos /* But we need to zero-extend (char is unsigned) the value and then @@ -159,22 +167,12 @@ L(mutual_align): neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */ ldr data2, [src2], #8 mov tmp2, #~0 - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ -#ifdef __AARCH64EB__ - /* Big-endian. Early bytes are at MSB. */ - lsl tmp2, tmp2, tmp3 /* Shift (count & 63). */ -#else - /* Little-endian. Early bytes are at LSB. */ - lsr tmp2, tmp2, tmp3 /* Shift (count & 63). */ -#endif - and tmp3, limit_wd, #7 - lsr limit_wd, limit_wd, #3 - /* Adjust the limit. Only low 3 bits used, so overflow irrelevant. */ - add limit, limit, count - add tmp3, tmp3, count + LS_FW tmp2, tmp2, tmp3 /* Shift (count & 63). */ + /* Adjust the limit and ensure it doesn't overflow. */ + adds limit, limit, count + csinv limit, limit, xzr, lo orr data1, data1, tmp2 orr data2, data2, tmp2 - add limit_wd, limit_wd, tmp3, lsr #3 b L(start_realigned) .p2align 4 @@ -197,13 +195,11 @@ L(done): /* Align the SRC1 to a dword by doing a bytewise compare and then do the dword loop. */ L(try_misaligned_words): - lsr limit_wd, limit, #3 - cbz count, L(do_misaligned) + cbz count, L(src1_aligned) neg count, count and count, count, #7 sub limit, limit, count - lsr limit_wd, limit, #3 L(page_end_loop): ldrb data1w, [src1], #1 @@ -214,48 +210,101 @@ L(page_end_loop): subs count, count, #1 b.hi L(page_end_loop) -L(do_misaligned): - /* Prepare ourselves for the next page crossing. Unlike the aligned - loop, we fetch 1 less dword because we risk crossing bounds on - SRC2. */ - mov count, #8 - subs limit_wd, limit_wd, #1 - b.lo L(done_loop) -L(loop_misaligned): - and tmp2, src2, #0xff8 - eor tmp2, tmp2, #0xff8 - cbz tmp2, L(page_end_loop) + /* The following diagram explains the comparison of misaligned strings. + The bytes are shown in natural order. For little-endian, it is + reversed in the registers. The "x" bytes are before the string. + The "|" separates data that is loaded at one time. + src1 | a a a a a a a a | b b b c c c c c | . . . + src2 | x x x x x a a a a a a a a b b b | c c c c c . . . + + After shifting in each step, the data looks like this: + STEP_A STEP_B STEP_C + data1 a a a a a a a a b b b c c c c c b b b c c c c c + data2 a a a a a a a a b b b 0 0 0 0 0 0 0 0 c c c c c + The bytes with "0" are eliminated from the syndrome via mask. + + Align SRC2 down to 16 bytes. This way we can read 16 bytes at a + time from SRC2. The comparison happens in 3 steps. After each step + the loop can exit, or read from SRC1 or SRC2. */ +L(src1_aligned): + /* Calculate offset from 8 byte alignment to string start in bits. No + need to mask offset since shifts are ignoring upper bits. */ + lsl offset, src2, #3 + bic src2, src2, #0xf + mov mask, -1 + neg neg_offset, offset ldr data1, [src1], #8 - ldr data2, [src2], #8 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ - ccmp diff, #0, #0, eq - b.ne L(not_limit) - subs limit_wd, limit_wd, #1 - b.pl L(loop_misaligned) + ldp tmp1, tmp2, [src2], #16 + LS_BK mask, mask, neg_offset + and neg_offset, neg_offset, #63 /* Need actual value for cmp later. */ + /* Skip the first compare if data in tmp1 is irrelevant. */ + tbnz offset, 6, L(misaligned_mid_loop) -L(done_loop): - /* We found a difference or a NULL before the limit was reached. */ - and limit, limit, #7 - cbz limit, L(not_limit) - /* Read the last word. */ - sub src1, src1, 8 - sub src2, src2, 8 - ldr data1, [src1, limit] - ldr data2, [src2, limit] - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f +L(loop_misaligned): + /* STEP_A: Compare full 8 bytes when there is enough data from SRC2.*/ + LS_FW data2, tmp1, offset + LS_BK tmp1, tmp2, neg_offset + subs limit, limit, #8 + orr data2, data2, tmp1 /* 8 bytes from SRC2 combined from two regs.*/ + sub has_nul, data1, zeroones eor diff, data1, data2 /* Non-zero if differences found. */ - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ - ccmp diff, #0, #0, eq - b.ne L(not_limit) + orr tmp3, data1, #REP8_7f + csinv endloop, diff, xzr, hi /* If limit, set to all ones. */ + bic has_nul, has_nul, tmp3 /* Non-zero if NUL byte found in SRC1. */ + orr tmp3, endloop, has_nul + cbnz tmp3, L(full_check) + + ldr data1, [src1], #8 +L(misaligned_mid_loop): + /* STEP_B: Compare first part of data1 to second part of tmp2. */ + LS_FW data2, tmp2, offset +#ifdef __AARCH64EB__ + /* For big-endian we do a byte reverse to avoid carry-propagation + problem described above. This way we can reuse the has_nul in the + next step and also use syndrome value trick at the end. */ + rev tmp3, data1 + #define data1_fixed tmp3 +#else + #define data1_fixed data1 +#endif + sub has_nul, data1_fixed, zeroones + orr tmp3, data1_fixed, #REP8_7f + eor diff, data2, data1 /* Non-zero if differences found. */ + bic has_nul, has_nul, tmp3 /* Non-zero if NUL terminator. */ +#ifdef __AARCH64EB__ + rev has_nul, has_nul +#endif + cmp limit, neg_offset, lsr #3 + orr syndrome, diff, has_nul + bic syndrome, syndrome, mask /* Ignore later bytes. */ + csinv tmp3, syndrome, xzr, hi /* If limit, set to all ones. */ + cbnz tmp3, L(syndrome_check) + + /* STEP_C: Compare second part of data1 to first part of tmp1. */ + ldp tmp1, tmp2, [src2], #16 + cmp limit, #8 + LS_BK data2, tmp1, neg_offset + eor diff, data2, data1 /* Non-zero if differences found. */ + orr syndrome, diff, has_nul + and syndrome, syndrome, mask /* Ignore earlier bytes. */ + csinv tmp3, syndrome, xzr, hi /* If limit, set to all ones. */ + cbnz tmp3, L(syndrome_check) + + ldr data1, [src1], #8 + sub limit, limit, #8 + b L(loop_misaligned) + +#ifdef __AARCH64EB__ +L(syndrome_check): + clz pos, syndrome + cmp pos, limit, lsl #3 + b.lo L(end_quick) +#endif L(ret0): mov result, #0 ret - -SYM_FUNC_END_PI(strncmp) -EXPORT_SYMBOL_NOHWKASAN(strncmp) +SYM_FUNC_END(__pi_strncmp) +SYM_FUNC_ALIAS_WEAK(strncmp, __pi_strncmp) +EXPORT_SYMBOL_NOKASAN(strncmp) diff --git a/arch/arm64/lib/strnlen.S b/arch/arm64/lib/strnlen.S index b72913a99038..d5ac0e10a01d 100644 --- a/arch/arm64/lib/strnlen.S +++ b/arch/arm64/lib/strnlen.S @@ -47,7 +47,7 @@ limit_wd .req x14 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 -SYM_FUNC_START_WEAK_PI(strnlen) +SYM_FUNC_START(__pi_strnlen) cbz limit, .Lhit_limit mov zeroones, #REP8_01 bic src, srcin, #15 @@ -156,5 +156,7 @@ CPU_LE( lsr tmp2, tmp2, tmp4 ) /* Shift (tmp1 & 63). */ .Lhit_limit: mov len, limit ret -SYM_FUNC_END_PI(strnlen) +SYM_FUNC_END(__pi_strnlen) + +SYM_FUNC_ALIAS_WEAK(strnlen, __pi_strnlen) EXPORT_SYMBOL_NOKASAN(strnlen) diff --git a/arch/arm64/lib/strrchr.S b/arch/arm64/lib/strrchr.S index 13132d1ed6d1..a5123cf0ce12 100644 --- a/arch/arm64/lib/strrchr.S +++ b/arch/arm64/lib/strrchr.S @@ -18,7 +18,7 @@ * Returns: * x0 - address of last occurrence of 'c' or 0 */ -SYM_FUNC_START_WEAK_PI(strrchr) +SYM_FUNC_START(__pi_strrchr) mov x3, #0 and w1, w1, #0xff 1: ldrb w2, [x0], #1 @@ -29,5 +29,6 @@ SYM_FUNC_START_WEAK_PI(strrchr) b 1b 2: mov x0, x3 ret -SYM_FUNC_END_PI(strrchr) +SYM_FUNC_END(__pi_strrchr) +SYM_FUNC_ALIAS_WEAK(strrchr, __pi_strrchr) EXPORT_SYMBOL_NOKASAN(strrchr) diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c index d189cf4e70ea..96b171995d19 100644 --- a/arch/arm64/lib/xor-neon.c +++ b/arch/arm64/lib/xor-neon.c @@ -10,8 +10,8 @@ #include <linux/module.h> #include <asm/neon-intrinsics.h> -void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, - unsigned long *p2) +void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -37,8 +37,9 @@ void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3) +void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -72,8 +73,10 @@ void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3, unsigned long *p4) +void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -115,9 +118,11 @@ void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3, - unsigned long *p4, unsigned long *p5) +void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -186,8 +191,10 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) return res; } -static void xor_arm64_eor3_3(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3) +static void xor_arm64_eor3_3(unsigned long bytes, + unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -219,9 +226,11 @@ static void xor_arm64_eor3_3(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -static void xor_arm64_eor3_4(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3, - unsigned long *p4) +static void xor_arm64_eor3_4(unsigned long bytes, + unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; @@ -261,9 +270,12 @@ static void xor_arm64_eor3_4(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -static void xor_arm64_eor3_5(unsigned long bytes, unsigned long *p1, - unsigned long *p2, unsigned long *p3, - unsigned long *p4, unsigned long *p5) +static void xor_arm64_eor3_5(unsigned long bytes, + unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { uint64_t *dp1 = (uint64_t *)p1; uint64_t *dp2 = (uint64_t *)p2; diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 7d0563db4201..0ea6cc25dc66 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -107,10 +107,11 @@ SYM_FUNC_END(icache_inval_pou) * - start - virtual start address of region * - end - virtual end address of region */ -SYM_FUNC_START_PI(dcache_clean_inval_poc) +SYM_FUNC_START(__pi_dcache_clean_inval_poc) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(dcache_clean_inval_poc) +SYM_FUNC_END(__pi_dcache_clean_inval_poc) +SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc) /* * dcache_clean_pou(start, end) @@ -140,7 +141,7 @@ SYM_FUNC_END(dcache_clean_pou) * - start - kernel start address of region * - end - kernel end address of region */ -SYM_FUNC_START_PI(dcache_inval_poc) +SYM_FUNC_START(__pi_dcache_inval_poc) dcache_line_size x2, x3 sub x3, x2, #1 tst x1, x3 // end cache line aligned? @@ -158,7 +159,8 @@ SYM_FUNC_START_PI(dcache_inval_poc) b.lo 2b dsb sy ret -SYM_FUNC_END_PI(dcache_inval_poc) +SYM_FUNC_END(__pi_dcache_inval_poc) +SYM_FUNC_ALIAS(dcache_inval_poc, __pi_dcache_inval_poc) /* * dcache_clean_poc(start, end) @@ -169,10 +171,11 @@ SYM_FUNC_END_PI(dcache_inval_poc) * - start - virtual start address of region * - end - virtual end address of region */ -SYM_FUNC_START_PI(dcache_clean_poc) +SYM_FUNC_START(__pi_dcache_clean_poc) dcache_by_line_op cvac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(dcache_clean_poc) +SYM_FUNC_END(__pi_dcache_clean_poc) +SYM_FUNC_ALIAS(dcache_clean_poc, __pi_dcache_clean_poc) /* * dcache_clean_pop(start, end) @@ -183,13 +186,14 @@ SYM_FUNC_END_PI(dcache_clean_poc) * - start - virtual start address of region * - end - virtual end address of region */ -SYM_FUNC_START_PI(dcache_clean_pop) +SYM_FUNC_START(__pi_dcache_clean_pop) alternative_if_not ARM64_HAS_DCPOP b dcache_clean_poc alternative_else_nop_endif dcache_by_line_op cvap, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(dcache_clean_pop) +SYM_FUNC_END(__pi_dcache_clean_pop) +SYM_FUNC_ALIAS(dcache_clean_pop, __pi_dcache_clean_pop) /* * __dma_flush_area(start, size) @@ -199,11 +203,12 @@ SYM_FUNC_END_PI(dcache_clean_pop) * - start - virtual start address of region * - size - size in question */ -SYM_FUNC_START_PI(__dma_flush_area) +SYM_FUNC_START(__pi___dma_flush_area) add x1, x0, x1 dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(__dma_flush_area) +SYM_FUNC_END(__pi___dma_flush_area) +SYM_FUNC_ALIAS(__dma_flush_area, __pi___dma_flush_area) /* * __dma_map_area(start, size, dir) @@ -211,12 +216,13 @@ SYM_FUNC_END_PI(__dma_flush_area) * - size - size of region * - dir - DMA direction */ -SYM_FUNC_START_PI(__dma_map_area) +SYM_FUNC_START(__pi___dma_map_area) add x1, x0, x1 cmp w2, #DMA_FROM_DEVICE b.eq __pi_dcache_inval_poc b __pi_dcache_clean_poc -SYM_FUNC_END_PI(__dma_map_area) +SYM_FUNC_END(__pi___dma_map_area) +SYM_FUNC_ALIAS(__dma_map_area, __pi___dma_map_area) /* * __dma_unmap_area(start, size, dir) @@ -224,9 +230,10 @@ SYM_FUNC_END_PI(__dma_map_area) * - size - size of region * - dir - DMA direction */ -SYM_FUNC_START_PI(__dma_unmap_area) +SYM_FUNC_START(__pi___dma_unmap_area) add x1, x0, x1 cmp w2, #DMA_TO_DEVICE b.ne __pi_dcache_inval_poc ret -SYM_FUNC_END_PI(__dma_unmap_area) +SYM_FUNC_END(__pi___dma_unmap_area) +SYM_FUNC_ALIAS(__dma_unmap_area, __pi___dma_unmap_area) diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 2aaf950b906c..a06c6ac770d4 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -52,6 +52,13 @@ void __sync_icache_dcache(pte_t pte) { struct page *page = pte_page(pte); + /* + * HugeTLB pages are always fully mapped, so only setting head page's + * PG_dcache_clean flag is enough. + */ + if (PageHuge(page)) + page = compound_head(page); + if (!test_bit(PG_dcache_clean, &page->flags)) { sync_icache_aliases((unsigned long)page_address(page), (unsigned long)page_address(page) + diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index ffb9c229610a..cbace1c9e137 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -56,25 +56,34 @@ void __init arm64_hugetlb_cma_reserve(void) } #endif /* CONFIG_CMA */ -#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION -bool arch_hugetlb_migration_supported(struct hstate *h) +static bool __hugetlb_valid_size(unsigned long size) { - size_t pagesize = huge_page_size(h); - - switch (pagesize) { + switch (size) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: return pud_sect_supported(); #endif - case PMD_SIZE: case CONT_PMD_SIZE: + case PMD_SIZE: case CONT_PTE_SIZE: return true; } - pr_warn("%s: unrecognized huge page size 0x%lx\n", - __func__, pagesize); + return false; } + +#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION +bool arch_hugetlb_migration_supported(struct hstate *h) +{ + size_t pagesize = huge_page_size(h); + + if (!__hugetlb_valid_size(pagesize)) { + pr_warn("%s: unrecognized huge page size 0x%lx\n", + __func__, pagesize); + return false; + } + return true; +} #endif int pmd_huge(pmd_t pmd) @@ -347,6 +356,7 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) { size_t pagesize = 1UL << shift; + entry = pte_mkhuge(entry); if (pagesize == CONT_PTE_SIZE) { entry = pte_mkcont(entry); } else if (pagesize == CONT_PMD_SIZE) { @@ -506,16 +516,5 @@ arch_initcall(hugetlbpage_init); bool __init arch_hugetlb_valid_size(unsigned long size) { - switch (size) { -#ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: - return pud_sect_supported(); -#endif - case CONT_PMD_SIZE: - case PMD_SIZE: - case CONT_PTE_SIZE: - return true; - } - - return false; + return __hugetlb_valid_size(size); } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index db63cc885771..9e26ec80d317 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -61,8 +61,34 @@ EXPORT_SYMBOL(memstart_addr); * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4). * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory, * otherwise it is empty. + * + * Memory reservation for crash kernel either done early or deferred + * depending on DMA memory zones configs (ZONE_DMA) -- + * + * In absence of ZONE_DMA configs arm64_dma_phys_limit initialized + * here instead of max_zone_phys(). This lets early reservation of + * crash kernel memory which has a dependency on arm64_dma_phys_limit. + * Reserving memory early for crash kernel allows linear creation of block + * mappings (greater than page-granularity) for all the memory bank rangs. + * In this scheme a comparatively quicker boot is observed. + * + * If ZONE_DMA configs are defined, crash kernel memory reservation + * is delayed until DMA zone memory range size initilazation performed in + * zone_sizes_init(). The defer is necessary to steer clear of DMA zone + * memory range to avoid overlap allocation. So crash kernel memory boundaries + * are not known when mapping all bank memory ranges, which otherwise means + * not possible to exclude crash kernel range from creating block mappings + * so page-granularity mappings are created for the entire memory range. + * Hence a slightly slower boot is observed. + * + * Note: Page-granularity mapppings are necessary for crash kernel memory + * range for shrinking its size via /sys/kernel/kexec_crash_size interface. */ -phys_addr_t arm64_dma_phys_limit __ro_after_init; +#if IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32) +phys_addr_t __ro_after_init arm64_dma_phys_limit; +#else +phys_addr_t __ro_after_init arm64_dma_phys_limit = PHYS_MASK + 1; +#endif #ifdef CONFIG_KEXEC_CORE /* @@ -153,8 +179,6 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) if (!arm64_dma_phys_limit) arm64_dma_phys_limit = dma32_phys_limit; #endif - if (!arm64_dma_phys_limit) - arm64_dma_phys_limit = PHYS_MASK + 1; max_zone_pfns[ZONE_NORMAL] = max; free_area_init(max_zone_pfns); @@ -315,6 +339,9 @@ void __init arm64_memblock_init(void) early_init_fdt_scan_reserved_mem(); + if (!IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32)) + reserve_crashkernel(); + high_memory = __va(memblock_end_of_DRAM() - 1) + 1; } @@ -361,7 +388,8 @@ void __init bootmem_init(void) * request_standard_resources() depends on crashkernel's memory being * reserved, so do it here. */ - reserve_crashkernel(); + if (IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32)) + reserve_crashkernel(); memblock_dump_all(); } diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index a38f54cd638c..77ada00280d9 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -7,8 +7,10 @@ #include <linux/io.h> #include <linux/memblock.h> +#include <linux/mm.h> #include <linux/types.h> +#include <asm/cpufeature.h> #include <asm/page.h> /* @@ -38,3 +40,18 @@ int valid_mmap_phys_addr_range(unsigned long pfn, size_t size) { return !(((pfn << PAGE_SHIFT) + size) & ~PHYS_MASK); } + +static int __init adjust_protection_map(void) +{ + /* + * With Enhanced PAN we can honour the execute-only permissions as + * there is no PAN override with such mappings. + */ + if (cpus_have_const_cap(ARM64_HAS_EPAN)) { + protection_map[VM_EXEC] = PAGE_EXECONLY; + protection_map[VM_EXEC | VM_SHARED] = PAGE_EXECONLY; + } + + return 0; +} +arch_initcall(adjust_protection_map); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index acfae9b41cc8..626ec32873c6 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -17,6 +17,7 @@ #include <linux/mman.h> #include <linux/nodemask.h> #include <linux/memblock.h> +#include <linux/memremap.h> #include <linux/memory.h> #include <linux/fs.h> #include <linux/io.h> @@ -63,6 +64,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; static DEFINE_SPINLOCK(swapper_pgdir_lock); +static DEFINE_MUTEX(fixmap_lock); void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) { @@ -294,18 +296,6 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, } while (addr = next, addr != end); } -static inline bool use_1G_block(unsigned long addr, unsigned long next, - unsigned long phys) -{ - if (PAGE_SHIFT != 12) - return false; - - if (((addr | next | phys) & ~PUD_MASK) != 0) - return false; - - return true; -} - static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(int), @@ -329,6 +319,12 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, } BUG_ON(p4d_bad(p4d)); + /* + * No need for locking during early boot. And it doesn't work as + * expected with KASLR enabled. + */ + if (system_state != SYSTEM_BOOTING) + mutex_lock(&fixmap_lock); pudp = pud_set_fixmap_offset(p4dp, addr); do { pud_t old_pud = READ_ONCE(*pudp); @@ -338,7 +334,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, /* * For 4K granule only, attempt to put down a 1GB block */ - if (use_1G_block(addr, next, phys) && + if (pud_sect_supported() && + ((addr | next | phys) & ~PUD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { pud_set_huge(pudp, phys, prot); @@ -359,6 +356,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, } while (pudp++, addr = next, addr != end); pud_clear_fixmap(); + if (system_state != SYSTEM_BOOTING) + mutex_unlock(&fixmap_lock); } static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, @@ -517,7 +516,7 @@ static void __init map_mem(pgd_t *pgdp) */ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); - if (can_set_direct_map() || crash_mem_map || IS_ENABLED(CONFIG_KFENCE)) + if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE)) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* @@ -528,6 +527,17 @@ static void __init map_mem(pgd_t *pgdp) */ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); +#ifdef CONFIG_KEXEC_CORE + if (crash_mem_map) { + if (IS_ENABLED(CONFIG_ZONE_DMA) || + IS_ENABLED(CONFIG_ZONE_DMA32)) + flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + else if (crashk_res.end) + memblock_mark_nomap(crashk_res.start, + resource_size(&crashk_res)); + } +#endif + /* map all the memory banks */ for_each_mem_range(i, &start, &end) { if (start >= end) @@ -554,6 +564,25 @@ static void __init map_mem(pgd_t *pgdp) __map_memblock(pgdp, kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); + + /* + * Use page-level mappings here so that we can shrink the region + * in page granularity and put back unused memory to buddy system + * through /sys/kernel/kexec_crash_size interface. + */ +#ifdef CONFIG_KEXEC_CORE + if (crash_mem_map && + !IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32)) { + if (crashk_res.end) { + __map_memblock(pgdp, crashk_res.start, + crashk_res.end + 1, + PAGE_KERNEL, + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); + memblock_clear_nomap(crashk_res.start, + resource_size(&crashk_res)); + } + } +#endif } void mark_rodata_ro(void) @@ -617,6 +646,8 @@ early_param("rodata", parse_rodata); #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 static int __init map_entry_trampoline(void) { + int i; + pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); @@ -625,11 +656,15 @@ static int __init map_entry_trampoline(void) /* Map only the text into the trampoline page table */ memset(tramp_pg_dir, 0, PGD_SIZE); - __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, - prot, __pgd_pgtable_alloc, 0); + __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, + entry_tramp_text_size(), prot, + __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); /* Map both the text and data into the kernel page table */ - __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); + for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) + __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, + pa_start + i * PAGE_SIZE, prot); + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { extern char __entry_tramp_data_start[]; diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index 7c4ef56265ee..a9e50e930484 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -12,7 +12,7 @@ static DEFINE_XARRAY(mte_pages); void *mte_allocate_tag_storage(void) { /* tags granule is 16 bytes, 2 tags stored per byte */ - return kmalloc(PAGE_SIZE / 16 / 2, GFP_KERNEL); + return kmalloc(MTE_PAGE_TAG_STORAGE, GFP_KERNEL); } void mte_free_tag_storage(char *storage) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index d35c90d2e47a..50bbed947bec 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -46,7 +46,7 @@ #endif #ifdef CONFIG_KASAN_HW_TAGS -#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 +#define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1 #else /* * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index cc0cf0f5c7c3..9d9250c7cc72 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -89,9 +89,16 @@ #define A64_STXR(sf, Rt, Rn, Rs) \ A64_LSX(sf, Rt, Rn, Rs, STORE_EX) -/* LSE atomics */ +/* + * LSE atomics + * + * STADD is simply encoded as an alias for LDADD with XZR as + * the destination register. + */ #define A64_STADD(sf, Rn, Rs) \ - aarch64_insn_gen_stadd(Rn, Rs, A64_SIZE(sf)) + aarch64_insn_gen_atomic_ld_op(A64_ZR, Rn, Rs, \ + A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_ADD, \ + AARCH64_INSN_MEM_ORDER_NONE) /* Add/subtract (immediate) */ #define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type) \ diff --git a/arch/arm64/tools/Makefile b/arch/arm64/tools/Makefile index 932b4fe5c768..cf1307188150 100644 --- a/arch/arm64/tools/Makefile +++ b/arch/arm64/tools/Makefile @@ -5,18 +5,14 @@ kapi := $(gen)/asm kapi-hdrs-y := $(kapi)/cpucaps.h -targets += $(addprefix ../../../,$(gen-y) $(kapi-hdrs-y)) +targets += $(addprefix ../../../, $(kapi-hdrs-y)) PHONY += kapi -kapi: $(kapi-hdrs-y) $(gen-y) - -# Create output directory if not already present -_dummy := $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +kapi: $(kapi-hdrs-y) quiet_cmd_gen_cpucaps = GEN $@ - cmd_gen_cpucaps = mkdir -p $(dir $@) && \ - $(AWK) -f $(filter-out $(PHONY),$^) > $@ + cmd_gen_cpucaps = mkdir -p $(dir $@); $(AWK) -f $(real-prereqs) > $@ $(kapi)/cpucaps.h: $(src)/gen-cpucaps.awk $(src)/cpucaps FORCE $(call if_changed,gen_cpucaps) diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9c65b1e25a96..3ed418f70e3b 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -7,7 +7,8 @@ BTI HAS_32BIT_EL0_DO_NOT_USE HAS_32BIT_EL1 HAS_ADDRESS_AUTH -HAS_ADDRESS_AUTH_ARCH +HAS_ADDRESS_AUTH_ARCH_QARMA3 +HAS_ADDRESS_AUTH_ARCH_QARMA5 HAS_ADDRESS_AUTH_IMP_DEF HAS_AMU_EXTN HAS_ARMv8_4_TTL @@ -21,7 +22,8 @@ HAS_E0PD HAS_ECV HAS_EPAN HAS_GENERIC_AUTH -HAS_GENERIC_AUTH_ARCH +HAS_GENERIC_AUTH_ARCH_QARMA3 +HAS_GENERIC_AUTH_ARCH_QARMA5 HAS_GENERIC_AUTH_IMP_DEF HAS_IRQ_PRIO_MASKING HAS_LDAPR @@ -44,6 +46,7 @@ MTE_ASYMM SPECTRE_V2 SPECTRE_V3A SPECTRE_V4 +SPECTRE_BHB SSBS SVE UNMAP_KERNEL_AT_EL0 diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 132f43f12dd8..75ef86605d69 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -79,7 +79,6 @@ config CSKY select PCI_DOMAINS_GENERIC if PCI select PCI_SYSCALL if PCI select PCI_MSI if PCI - select SET_FS select TRACE_IRQFLAGS_SUPPORT config LOCKDEP_SUPPORT diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index 151607ed5158..bbe245117777 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -30,6 +30,7 @@ #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) +#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #define pte_clear(mm, addr, ptep) set_pte((ptep), \ (((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 817dd60ff152..688c7548b559 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -4,7 +4,6 @@ #define __ASM_CSKY_PROCESSOR_H #include <linux/bitops.h> -#include <asm/segment.h> #include <asm/ptrace.h> #include <asm/current.h> #include <asm/cache.h> @@ -59,7 +58,6 @@ struct thread_struct { */ #define start_thread(_regs, _pc, _usp) \ do { \ - set_fs(USER_DS); /* reads from user space */ \ (_regs)->pc = (_pc); \ (_regs)->regs[1] = 0; /* ABIV1 is R7, uClibc_main rtdl arg */ \ (_regs)->regs[2] = 0; \ diff --git a/arch/csky/include/asm/segment.h b/arch/csky/include/asm/segment.h deleted file mode 100644 index 5bc1cc62b87f..000000000000 --- a/arch/csky/include/asm/segment.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef __ASM_CSKY_SEGMENT_H -#define __ASM_CSKY_SEGMENT_H - -typedef struct { - unsigned long seg; -} mm_segment_t; - -#endif /* __ASM_CSKY_SEGMENT_H */ diff --git a/arch/csky/include/asm/thread_info.h b/arch/csky/include/asm/thread_info.h index 8c349a8f904d..b5ed788f0c68 100644 --- a/arch/csky/include/asm/thread_info.h +++ b/arch/csky/include/asm/thread_info.h @@ -16,7 +16,6 @@ struct thread_info { unsigned long flags; int preempt_count; unsigned long tp_value; - mm_segment_t addr_limit; struct restart_block restart_block; struct pt_regs *regs; unsigned int cpu; @@ -26,7 +25,6 @@ struct thread_info { { \ .task = &tsk, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ .cpu = 0, \ .restart_block = { \ .fn = do_no_restart_syscall, \ diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index c40f06ee8d3e..2e927c21d8a1 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -3,17 +3,6 @@ #ifndef __ASM_CSKY_UACCESS_H #define __ASM_CSKY_UACCESS_H -#define user_addr_max() \ - (uaccess_kernel() ? KERNEL_DS.seg : get_fs().seg) - -static inline int __access_ok(unsigned long addr, unsigned long size) -{ - unsigned long limit = current_thread_info()->addr_limit.seg; - - return ((addr < limit) && ((addr + size) < limit)); -} -#define __access_ok __access_ok - /* * __put_user_fn */ @@ -209,7 +198,6 @@ unsigned long raw_copy_to_user(void *to, const void *from, unsigned long n); unsigned long __clear_user(void __user *to, unsigned long n); #define __clear_user __clear_user -#include <asm/segment.h> #include <asm-generic/uaccess.h> #endif /* __ASM_CSKY_UACCESS_H */ diff --git a/arch/csky/kernel/asm-offsets.c b/arch/csky/kernel/asm-offsets.c index 1cbcba4b0dd1..d1e903579473 100644 --- a/arch/csky/kernel/asm-offsets.c +++ b/arch/csky/kernel/asm-offsets.c @@ -25,7 +25,6 @@ int main(void) /* offsets into the thread_info struct */ DEFINE(TINFO_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TINFO_PREEMPT, offsetof(struct thread_info, preempt_count)); - DEFINE(TINFO_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TINFO_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TINFO_TASK, offsetof(struct thread_info, task)); diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c index 92057de08f4f..1612f4354087 100644 --- a/arch/csky/kernel/perf_callchain.c +++ b/arch/csky/kernel/perf_callchain.c @@ -49,7 +49,7 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, { struct stackframe buftail; unsigned long lr = 0; - unsigned long *user_frame_tail = (unsigned long *)fp; + unsigned long __user *user_frame_tail = (unsigned long __user *)fp; /* Check accessibility of one struct frame_tail beyond */ if (!access_ok(user_frame_tail, sizeof(buftail))) diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c index c7b763d2f526..8867ddf3e6c7 100644 --- a/arch/csky/kernel/signal.c +++ b/arch/csky/kernel/signal.c @@ -136,7 +136,7 @@ static inline void __user *get_sigframe(struct ksignal *ksig, static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { - struct rt_sigframe *frame; + struct rt_sigframe __user *frame; int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); diff --git a/arch/h8300/Kconfig b/arch/h8300/Kconfig index 3e3e0f16f7e0..fe48c4f26cc8 100644 --- a/arch/h8300/Kconfig +++ b/arch/h8300/Kconfig @@ -24,7 +24,6 @@ config H8300 select HAVE_ARCH_KGDB select HAVE_ARCH_HASH select CPU_NO_EFFICIENT_FFS - select SET_FS select UACCESS_MEMCPY config CPU_BIG_ENDIAN diff --git a/arch/h8300/include/asm/processor.h b/arch/h8300/include/asm/processor.h index 141a23eb62b7..ba171aa4dacb 100644 --- a/arch/h8300/include/asm/processor.h +++ b/arch/h8300/include/asm/processor.h @@ -13,7 +13,6 @@ #define __ASM_H8300_PROCESSOR_H #include <linux/compiler.h> -#include <asm/segment.h> #include <asm/ptrace.h> #include <asm/current.h> diff --git a/arch/h8300/include/asm/segment.h b/arch/h8300/include/asm/segment.h deleted file mode 100644 index 37950725d9b9..000000000000 --- a/arch/h8300/include/asm/segment.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _H8300_SEGMENT_H -#define _H8300_SEGMENT_H - -/* define constants */ -#define USER_DATA (1) -#ifndef __USER_DS -#define __USER_DS (USER_DATA) -#endif -#define USER_PROGRAM (2) -#define SUPER_DATA (3) -#ifndef __KERNEL_DS -#define __KERNEL_DS (SUPER_DATA) -#endif -#define SUPER_PROGRAM (4) - -#ifndef __ASSEMBLY__ - -typedef struct { - unsigned long seg; -} mm_segment_t; - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) -#define USER_DS MAKE_MM_SEG(__USER_DS) -#define KERNEL_DS MAKE_MM_SEG(__KERNEL_DS) - -/* - * Get/set the SFC/DFC registers for MOVES instructions - */ - -static inline mm_segment_t get_fs(void) -{ - return USER_DS; -} - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -#endif /* __ASSEMBLY__ */ - -#endif /* _H8300_SEGMENT_H */ diff --git a/arch/h8300/include/asm/thread_info.h b/arch/h8300/include/asm/thread_info.h index a518214d4ddd..ff2d873749a4 100644 --- a/arch/h8300/include/asm/thread_info.h +++ b/arch/h8300/include/asm/thread_info.h @@ -10,7 +10,6 @@ #define _ASM_THREAD_INFO_H #include <asm/page.h> -#include <asm/segment.h> #ifdef __KERNEL__ @@ -31,7 +30,6 @@ struct thread_info { unsigned long flags; /* low level flags */ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ - mm_segment_t addr_limit; }; /* @@ -43,7 +41,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } /* how to get the thread information struct from C */ diff --git a/arch/h8300/include/uapi/asm/signal.h b/arch/h8300/include/uapi/asm/signal.h index 2cd0dce2b6a6..1165481f80f6 100644 --- a/arch/h8300/include/uapi/asm/signal.h +++ b/arch/h8300/include/uapi/asm/signal.h @@ -85,7 +85,7 @@ struct sigaction { typedef struct sigaltstack { void *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/h8300/kernel/entry.S b/arch/h8300/kernel/entry.S index c6e289b5f1f2..42db87c17917 100644 --- a/arch/h8300/kernel/entry.S +++ b/arch/h8300/kernel/entry.S @@ -17,7 +17,6 @@ #include <linux/sys.h> #include <asm/unistd.h> #include <asm/setup.h> -#include <asm/segment.h> #include <asm/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> diff --git a/arch/h8300/kernel/head_ram.S b/arch/h8300/kernel/head_ram.S index dbf8429f5fab..489462f0ee57 100644 --- a/arch/h8300/kernel/head_ram.S +++ b/arch/h8300/kernel/head_ram.S @@ -4,7 +4,6 @@ #include <linux/init.h> #include <asm/unistd.h> #include <asm/setup.h> -#include <asm/segment.h> #include <asm/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> diff --git a/arch/h8300/mm/init.c b/arch/h8300/mm/init.c index f7bf4693e3b2..9fa13312720a 100644 --- a/arch/h8300/mm/init.c +++ b/arch/h8300/mm/init.c @@ -34,7 +34,6 @@ #include <linux/gfp.h> #include <asm/setup.h> -#include <asm/segment.h> #include <asm/page.h> #include <asm/sections.h> @@ -71,11 +70,6 @@ void __init paging_init(void) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", __func__, PAGE_SIZE, PAGE_SIZE); - /* - * Set up SFC/DFC registers (user data space). - */ - set_fs(USER_DS); - pr_debug("before free_area_init\n"); pr_debug("free_area_init -> start_mem is %#lx\nvirtual_end is %#lx\n", diff --git a/arch/h8300/mm/memory.c b/arch/h8300/mm/memory.c index 4a60e2b5eb96..c950571064d2 100644 --- a/arch/h8300/mm/memory.c +++ b/arch/h8300/mm/memory.c @@ -24,7 +24,6 @@ #include <linux/types.h> #include <asm/setup.h> -#include <asm/segment.h> #include <asm/page.h> #include <asm/traps.h> #include <asm/io.h> diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 15dd8f38b698..54eadf265178 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -30,7 +30,6 @@ config HEXAGON select GENERIC_CLOCKEVENTS_BROADCAST select MODULES_USE_ELF_RELA select GENERIC_CPU_DEVICES - select SET_FS select ARCH_WANT_LD_ORPHAN_WARN select TRACE_IRQFLAGS_SUPPORT help diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h index 18cd6ea9ab23..0610724d6a28 100644 --- a/arch/hexagon/include/asm/pgtable.h +++ b/arch/hexagon/include/asm/pgtable.h @@ -236,6 +236,11 @@ static inline int pmd_bad(pmd_t pmd) } /* + * pmd_pfn - converts a PMD entry to a page frame number + */ +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) + +/* * pmd_page - converts a PMD entry to a page pointer */ #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) diff --git a/arch/hexagon/include/asm/thread_info.h b/arch/hexagon/include/asm/thread_info.h index 535976665bf0..e90f280b9ce3 100644 --- a/arch/hexagon/include/asm/thread_info.h +++ b/arch/hexagon/include/asm/thread_info.h @@ -22,10 +22,6 @@ #ifndef __ASSEMBLY__ -typedef struct { - unsigned long seg; -} mm_segment_t; - /* * This is union'd with the "bottom" of the kernel stack. * It keeps track of thread info which is handy for routines @@ -37,7 +33,6 @@ struct thread_info { unsigned long flags; /* low level flags */ __u32 cpu; /* current cpu */ int preempt_count; /* 0=>preemptible,<0=>BUG */ - mm_segment_t addr_limit; /* segmentation sux */ /* * used for syscalls somehow; * seems to have a function pointer and four arguments @@ -66,7 +61,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = 1, \ - .addr_limit = KERNEL_DS, \ .sp = 0, \ .regs = NULL, \ } diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index ef5bfef8d490..bff77efc0d9a 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -13,31 +13,6 @@ #include <asm/sections.h> /* - * access_ok: - Checks if a user space pointer is valid - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block *may* be valid, false (zero) - * if it is definitely invalid. - * - * User address space in Hexagon, like x86, goes to 0xbfffffff, so the - * simple MSB-based tests used by MIPS won't work. Some further - * optimization is probably possible here, but for now, keep it - * reasonably simple and not *too* slow. After all, we've got the - * MMU for backup. - */ - -#define __access_ok(addr, size) \ - ((get_fs().seg == KERNEL_DS.seg) || \ - (((unsigned long)addr < get_fs().seg) && \ - (unsigned long)size < (get_fs().seg - (unsigned long)addr))) - -/* * When a kernel-mode page fault is taken, the faulting instruction * address is checked against a table of exception_table_entries. * Each entry is a tuple of the address of an instruction that may diff --git a/arch/hexagon/kernel/process.c b/arch/hexagon/kernel/process.c index 232dfd8956aa..dfa6b2757c05 100644 --- a/arch/hexagon/kernel/process.c +++ b/arch/hexagon/kernel/process.c @@ -105,7 +105,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, /* * Parent sees new pid -- not necessary, not even possible at * this point in the fork process - * Might also want to set things like ti->addr_limit */ return 0; diff --git a/arch/hexagon/mm/init.c b/arch/hexagon/mm/init.c index f01e91e10d95..3167a3b5c97b 100644 --- a/arch/hexagon/mm/init.c +++ b/arch/hexagon/mm/init.c @@ -29,8 +29,6 @@ int max_kernel_seg = 0x303; /* indicate pfn's of high memory */ unsigned long highstart_pfn, highend_pfn; -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); - /* Default cache attribute for newly created page tables */ unsigned long _dflt_cache_att = CACHEDEF; diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index a7e01573abd8..b2317324827f 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -8,6 +8,7 @@ menu "Processor type and features" config IA64 bool + select ARCH_BINFMT_ELF_EXTRA_PHDRS select ARCH_HAS_DMA_MARK_CLEAN select ARCH_HAS_STRNCPY_FROM_USER select ARCH_HAS_STRNLEN_USER @@ -61,7 +62,6 @@ config IA64 select NEED_SG_DMA_LENGTH select NUMA if !FLATMEM select PCI_MSI_ARCH_FALLBACKS if PCI_MSI - select SET_FS select ZONE_DMA32 default y help diff --git a/arch/ia64/configs/zx1_defconfig b/arch/ia64/configs/zx1_defconfig index 629cb9cdf723..851d8594cdb8 100644 --- a/arch/ia64/configs/zx1_defconfig +++ b/arch/ia64/configs/zx1_defconfig @@ -106,7 +106,6 @@ CONFIG_HUGETLBFS=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=y CONFIG_NLS_CODEPAGE_775=y diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 9584b2c5f394..7aa8f2330fb1 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -267,6 +267,7 @@ ia64_phys_addr_valid (unsigned long addr) #define pmd_present(pmd) (pmd_val(pmd) != 0UL) #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & _PFN_MASK)) +#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> PAGE_SHIFT) #define pmd_page(pmd) virt_to_page((pmd_val(pmd) + PAGE_OFFSET)) #define pud_none(pud) (!pud_val(pud)) diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 45365c2ef598..7cbce290f4e5 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -243,10 +243,6 @@ DECLARE_PER_CPU(struct cpuinfo_ia64, ia64_cpu_info); extern void print_cpu_info (struct cpuinfo_ia64 *); -typedef struct { - unsigned long seg; -} mm_segment_t; - #define SET_UNALIGN_CTL(task,value) \ ({ \ (task)->thread.flags = (((task)->thread.flags & ~IA64_THREAD_UAC_MASK) \ diff --git a/arch/ia64/include/asm/thread_info.h b/arch/ia64/include/asm/thread_info.h index 51d20cb37706..21b257117e0a 100644 --- a/arch/ia64/include/asm/thread_info.h +++ b/arch/ia64/include/asm/thread_info.h @@ -27,7 +27,6 @@ struct thread_info { __u32 cpu; /* current CPU */ __u32 last_cpu; /* Last CPU thread ran on */ __u32 status; /* Thread synchronous flags */ - mm_segment_t addr_limit; /* user-level address space limit */ int preempt_count; /* 0=premptable, <0=BUG; will also serve as bh-counter */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE __u64 utime; @@ -48,22 +47,21 @@ struct thread_info { .task = &tsk, \ .flags = 0, \ .cpu = 0, \ - .addr_limit = KERNEL_DS, \ .preempt_count = INIT_PREEMPT_COUNT, \ } #ifndef ASM_OFFSETS_C /* how to get the thread information struct from C */ #define current_thread_info() ((struct thread_info *) ((char *) current + IA64_TASK_SIZE)) -#define alloc_thread_stack_node(tsk, node) \ +#define arch_alloc_thread_stack_node(tsk, node) \ ((unsigned long *) ((char *) (tsk) + IA64_TASK_SIZE)) #define task_thread_info(tsk) ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE)) #else #define current_thread_info() ((struct thread_info *) 0) -#define alloc_thread_stack_node(tsk, node) ((unsigned long *) 0) +#define arch_alloc_thread_stack_node(tsk, node) ((unsigned long *) 0) #define task_thread_info(tsk) ((struct thread_info *) 0) #endif -#define free_thread_stack(tsk) /* nothing */ +#define arch_free_thread_stack(tsk) /* nothing */ #define task_stack_page(tsk) ((void *)(tsk)) #define __HAVE_THREAD_FUNCTIONS diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index e19d2dcc0ced..60adadeb3e9e 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -42,30 +42,20 @@ #include <asm/extable.h> /* - * For historical reasons, the following macros are grossly misnamed: - */ -#define KERNEL_DS ((mm_segment_t) { ~0UL }) /* cf. access_ok() */ -#define USER_DS ((mm_segment_t) { TASK_SIZE-1 }) /* cf. access_ok() */ - -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -/* - * When accessing user memory, we need to make sure the entire area really is in - * user-level space. In order to do this efficiently, we make sure that the page at - * address TASK_SIZE is never valid. We also need to make sure that the address doesn't + * When accessing user memory, we need to make sure the entire area really is + * in user-level space. We also need to make sure that the address doesn't * point inside the virtually mapped linear page table. */ static inline int __access_ok(const void __user *p, unsigned long size) { + unsigned long limit = TASK_SIZE; unsigned long addr = (unsigned long)p; - unsigned long seg = get_fs().seg; - return likely(addr <= seg) && - (seg == KERNEL_DS.seg || likely(REGION_OFFSET(addr) < RGN_MAP_LIMIT)); + + return likely((size <= limit) && (addr <= (limit - size)) && + likely(REGION_OFFSET(addr) < RGN_MAP_LIMIT)); } -#define access_ok(addr, size) __access_ok((addr), (size)) +#define __access_ok __access_ok +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They automatically diff --git a/arch/ia64/include/asm/xor.h b/arch/ia64/include/asm/xor.h index 673051bf9d7d..6785f70d3208 100644 --- a/arch/ia64/include/asm/xor.h +++ b/arch/ia64/include/asm/xor.h @@ -4,13 +4,20 @@ */ -extern void xor_ia64_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_ia64_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_ia64_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_ia64_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +extern void xor_ia64_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +extern void xor_ia64_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +extern void xor_ia64_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +extern void xor_ia64_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); static struct xor_block_template xor_block_ia64 = { .name = "ia64", diff --git a/arch/ia64/include/uapi/asm/signal.h b/arch/ia64/include/uapi/asm/signal.h index 38166a88e4c9..63d574e802a2 100644 --- a/arch/ia64/include/uapi/asm/signal.h +++ b/arch/ia64/include/uapi/asm/signal.h @@ -90,7 +90,7 @@ struct siginfo; typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index e4992917a24b..94a848b06f15 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -70,16 +70,6 @@ static int __init topology_init(void) { int i, err = 0; -#ifdef CONFIG_NUMA - /* - * MCD - Do we want to register all ONLINE nodes, or all POSSIBLE nodes? - */ - for_each_online_node(i) { - if ((err = register_one_node(i))) - goto out; - } -#endif - sysfs_cpus = kcalloc(NR_CPUS, sizeof(struct ia64_cpu), GFP_KERNEL); if (!sysfs_cpus) panic("kzalloc in topology_init failed - NR_CPUS too big?"); diff --git a/arch/ia64/kernel/unaligned.c b/arch/ia64/kernel/unaligned.c index 6c1a8951dfbb..0acb5a0cd7ab 100644 --- a/arch/ia64/kernel/unaligned.c +++ b/arch/ia64/kernel/unaligned.c @@ -749,9 +749,25 @@ emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsi } } +static int emulate_store(unsigned long ifa, void *val, int len, bool kernel_mode) +{ + if (kernel_mode) + return copy_to_kernel_nofault((void *)ifa, val, len); + + return copy_to_user((void __user *)ifa, val, len); +} + +static int emulate_load(void *val, unsigned long ifa, int len, bool kernel_mode) +{ + if (kernel_mode) + return copy_from_kernel_nofault(val, (void *)ifa, len); + + return copy_from_user(val, (void __user *)ifa, len); +} static int -emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) +emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs, + bool kernel_mode) { unsigned int len = 1 << ld.x6_sz; unsigned long val = 0; @@ -774,7 +790,7 @@ emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) return -1; } /* this assumes little-endian byte-order: */ - if (copy_from_user(&val, (void __user *) ifa, len)) + if (emulate_load(&val, ifa, len, kernel_mode)) return -1; setreg(ld.r1, val, 0, regs); @@ -872,7 +888,8 @@ emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) } static int -emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) +emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs, + bool kernel_mode) { unsigned long r2; unsigned int len = 1 << ld.x6_sz; @@ -901,7 +918,7 @@ emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) } /* this assumes little-endian byte-order: */ - if (copy_to_user((void __user *) ifa, &r2, len)) + if (emulate_store(ifa, &r2, len, kernel_mode)) return -1; /* @@ -1021,7 +1038,7 @@ float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final) } static int -emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs) +emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs, bool kernel_mode) { struct ia64_fpreg fpr_init[2]; struct ia64_fpreg fpr_final[2]; @@ -1050,8 +1067,8 @@ emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs * This assumes little-endian byte-order. Note that there is no "ldfpe" * instruction: */ - if (copy_from_user(&fpr_init[0], (void __user *) ifa, len) - || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len)) + if (emulate_load(&fpr_init[0], ifa, len, kernel_mode) + || emulate_load(&fpr_init[1], (ifa + len), len, kernel_mode)) return -1; DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz); @@ -1126,7 +1143,8 @@ emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs static int -emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) +emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs, + bool kernel_mode) { struct ia64_fpreg fpr_init; struct ia64_fpreg fpr_final; @@ -1152,7 +1170,7 @@ emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) * See comments in ldX for descriptions on how the various loads are handled. */ if (ld.x6_op != 0x2) { - if (copy_from_user(&fpr_init, (void __user *) ifa, len)) + if (emulate_load(&fpr_init, ifa, len, kernel_mode)) return -1; DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz); @@ -1202,7 +1220,8 @@ emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) static int -emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) +emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs, + bool kernel_mode) { struct ia64_fpreg fpr_init; struct ia64_fpreg fpr_final; @@ -1244,7 +1263,7 @@ emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) DDUMP("fpr_init =", &fpr_init, len); DDUMP("fpr_final =", &fpr_final, len); - if (copy_to_user((void __user *) ifa, &fpr_final, len)) + if (emulate_store(ifa, &fpr_final, len, kernel_mode)) return -1; /* @@ -1295,7 +1314,6 @@ void ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) { struct ia64_psr *ipsr = ia64_psr(regs); - mm_segment_t old_fs = get_fs(); unsigned long bundle[2]; unsigned long opcode; const struct exception_table_entry *eh = NULL; @@ -1304,6 +1322,7 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) load_store_t insn; } u; int ret = -1; + bool kernel_mode = false; if (ia64_psr(regs)->be) { /* we don't support big-endian accesses */ @@ -1367,13 +1386,13 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) if (unaligned_dump_stack) dump_stack(); } - set_fs(KERNEL_DS); + kernel_mode = true; } DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n", regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it); - if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16)) + if (emulate_load(bundle, regs->cr_iip, 16, kernel_mode)) goto failure; /* @@ -1467,7 +1486,7 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) case LDCCLR_IMM_OP: case LDCNC_IMM_OP: case LDCCLRACQ_IMM_OP: - ret = emulate_load_int(ifa, u.insn, regs); + ret = emulate_load_int(ifa, u.insn, regs, kernel_mode); break; case ST_OP: @@ -1478,7 +1497,7 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) fallthrough; case ST_IMM_OP: case STREL_IMM_OP: - ret = emulate_store_int(ifa, u.insn, regs); + ret = emulate_store_int(ifa, u.insn, regs, kernel_mode); break; case LDF_OP: @@ -1486,21 +1505,21 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) case LDFCCLR_OP: case LDFCNC_OP: if (u.insn.x) - ret = emulate_load_floatpair(ifa, u.insn, regs); + ret = emulate_load_floatpair(ifa, u.insn, regs, kernel_mode); else - ret = emulate_load_float(ifa, u.insn, regs); + ret = emulate_load_float(ifa, u.insn, regs, kernel_mode); break; case LDF_IMM_OP: case LDFA_IMM_OP: case LDFCCLR_IMM_OP: case LDFCNC_IMM_OP: - ret = emulate_load_float(ifa, u.insn, regs); + ret = emulate_load_float(ifa, u.insn, regs, kernel_mode); break; case STF_OP: case STF_IMM_OP: - ret = emulate_store_float(ifa, u.insn, regs); + ret = emulate_store_float(ifa, u.insn, regs, kernel_mode); break; default: @@ -1521,7 +1540,6 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip); done: - set_fs(old_fs); /* restore original address limit */ return; failure: diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 791d4176e4a6..73d0db36edb6 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -608,17 +608,11 @@ void __init paging_init(void) zero_page_memmap_ptr = virt_to_page(ia64_imva(empty_zero_page)); } -#ifdef CONFIG_MEMORY_HOTPLUG -pg_data_t *arch_alloc_nodedata(int nid) +pg_data_t * __init arch_alloc_nodedata(int nid) { unsigned long size = compute_pernodesize(nid); - return kzalloc(size, GFP_KERNEL); -} - -void arch_free_nodedata(pg_data_t *pgdat) -{ - kfree(pgdat); + return memblock_alloc(size, SMP_CACHE_BYTES); } void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat) @@ -626,7 +620,6 @@ void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat) pgdat_list[update_node] = update_pgdat; scatter_node_data(); } -#endif #ifdef CONFIG_SPARSEMEM_VMEMMAP int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, diff --git a/arch/m68k/68000/dragen2.c b/arch/m68k/68000/dragen2.c index 62f10a9e1ab7..1a57eff28cfe 100644 --- a/arch/m68k/68000/dragen2.c +++ b/arch/m68k/68000/dragen2.c @@ -11,6 +11,7 @@ #include <linux/init.h> #include <asm/machdep.h> #include <asm/MC68VZ328.h> +#include "m68328.h" #include "screen.h" /***************************************************************************/ diff --git a/arch/m68k/68000/screen.h b/arch/m68k/68000/screen.h index 2089bdf02688..8f7418008908 100644 --- a/arch/m68k/68000/screen.h +++ b/arch/m68k/68000/screen.h @@ -1,4 +1,5 @@ /* Created with The GIMP */ +#ifdef CONFIG_INIT_LCD #define screen_width 320 #define screen_height 240 static unsigned char screen_bits[] = { @@ -802,3 +803,4 @@ static unsigned char screen_bits[] = { 0x93, 0x10, 0xe2, 0x11, 0x00, 0x94, 0x22, 0x52, 0x69, 0x53, 0x52, 0x45, 0x49, 0x22, 0xa4, 0x4a, 0x55, 0x29, 0x2a, 0xa4, 0x52, 0x42, 0xaa, 0xa5, 0x52, 0xa8, 0xaa, 0x55, 0x4a, 0xab, 0xa9, 0x4a, 0x54, 0x49, 0x32, 0x24 }; +#endif /* CONFIG_INIT_LCD */ diff --git a/arch/m68k/68000/ucsimm.c b/arch/m68k/68000/ucsimm.c index 7c6cbf643712..c54fde75eae8 100644 --- a/arch/m68k/68000/ucsimm.c +++ b/arch/m68k/68000/ucsimm.c @@ -16,19 +16,18 @@ #include "m68328.h" -unsigned char *cs8900a_hwaddr; static int errno; -_bsc0(char *, getserialnum) -_bsc1(unsigned char *, gethwaddr, int, a) -_bsc1(char *, getbenv, char *, a) +static _bsc0(char *, getserialnum) +static _bsc1(unsigned char *, gethwaddr, int, a) +static _bsc1(char *, getbenv, char *, a) void __init init_ucsimm(char *command, int size) { char *p; pr_info("uCsimm/uCdimm serial string [%s]\n", getserialnum()); - p = cs8900a_hwaddr = gethwaddr(0); + p = gethwaddr(0); pr_info("uCsimm/uCdimm hwaddr %pM\n", p); p = getbenv("APPEND"); if (p) diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 936e1803c7c7..936cce42ae9a 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -4,6 +4,7 @@ config M68K default y select ARCH_32BIT_OFF_T select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DMA_PREP_COHERENT if HAS_DMA && MMU && !COLDFIRE select ARCH_HAS_SYNC_DMA_FOR_DEVICE if HAS_DMA select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS @@ -17,7 +18,6 @@ config M68K select GENERIC_CPU_DEVICES select GENERIC_IOMAP select GENERIC_IRQ_SHOW - select HAVE_AOUT if MMU select HAVE_ASM_MODVERSIONS select HAVE_DEBUG_BUGVERBOSE select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_HAS_NO_UNALIGNED diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 0d00ef5117dc..16ea9a67723c 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -453,6 +453,7 @@ config CPU_HAS_NO_UNALIGNED config CPU_HAS_ADDRESS_SPACES bool + select ALTERNATE_USER_ADDRESS_SPACE config FPU bool diff --git a/arch/m68k/amiga/config.c b/arch/m68k/amiga/config.c index be2dfab48fd4..3137b45750df 100644 --- a/arch/m68k/amiga/config.c +++ b/arch/m68k/amiga/config.c @@ -37,6 +37,7 @@ #include <asm/irq.h> #include <asm/machdep.h> #include <asm/io.h> +#include <asm/config.h> static unsigned long amiga_model; diff --git a/arch/m68k/apollo/config.c b/arch/m68k/apollo/config.c index 581a5f68d102..42a8b8e2b664 100644 --- a/arch/m68k/apollo/config.c +++ b/arch/m68k/apollo/config.c @@ -16,6 +16,7 @@ #include <asm/apollohw.h> #include <asm/irq.h> #include <asm/machdep.h> +#include <asm/config.h> u_long sio01_physaddr; u_long sio23_physaddr; diff --git a/arch/m68k/atari/config.c b/arch/m68k/atari/config.c index 261a0f57cc9a..38a7c0578105 100644 --- a/arch/m68k/atari/config.c +++ b/arch/m68k/atari/config.c @@ -46,6 +46,7 @@ #include <asm/machdep.h> #include <asm/hwtest.h> #include <asm/io.h> +#include <asm/config.h> u_long atari_mch_cookie; EXPORT_SYMBOL(atari_mch_cookie); diff --git a/arch/m68k/atari/stdma.c b/arch/m68k/atari/stdma.c index ba65f942d0c7..ce6818eff75e 100644 --- a/arch/m68k/atari/stdma.c +++ b/arch/m68k/atari/stdma.c @@ -30,7 +30,6 @@ #include <linux/types.h> #include <linux/kdev_t.h> -#include <linux/genhd.h> #include <linux/sched.h> #include <linux/init.h> #include <linux/interrupt.h> diff --git a/arch/m68k/bvme6000/config.c b/arch/m68k/bvme6000/config.c index 0c6feafbbd11..3a1d90e399e0 100644 --- a/arch/m68k/bvme6000/config.c +++ b/arch/m68k/bvme6000/config.c @@ -23,7 +23,6 @@ #include <linux/linkage.h> #include <linux/init.h> #include <linux/major.h> -#include <linux/genhd.h> #include <linux/rtc.h> #include <linux/interrupt.h> #include <linux/bcd.h> @@ -36,6 +35,7 @@ #include <asm/traps.h> #include <asm/machdep.h> #include <asm/bvme6000hw.h> +#include <asm/config.h> static void bvme6000_get_model(char *model); extern void bvme6000_sched_init(void); diff --git a/arch/m68k/coldfire/device.c b/arch/m68k/coldfire/device.c index 0386252e9d04..4218750414bb 100644 --- a/arch/m68k/coldfire/device.c +++ b/arch/m68k/coldfire/device.c @@ -480,7 +480,7 @@ static struct platform_device mcf_i2c5 = { #endif /* MCFI2C_BASE5 */ #endif /* IS_ENABLED(CONFIG_I2C_IMX) */ -#if IS_ENABLED(CONFIG_MCF_EDMA) +#ifdef MCFEDMA_BASE static const struct dma_slave_map mcf_edma_map[] = { { "dreq0", "rx-tx", MCF_EDMA_FILTER_PARAM(0) }, @@ -552,7 +552,7 @@ static struct platform_device mcf_edma = { .platform_data = &mcf_edma_data, } }; -#endif /* IS_ENABLED(CONFIG_MCF_EDMA) */ +#endif /* MCFEDMA_BASE */ #ifdef MCFSDHC_BASE static struct mcf_esdhc_platform_data mcf_esdhc_data = { @@ -651,7 +651,7 @@ static struct platform_device *mcf_devices[] __initdata = { &mcf_i2c5, #endif #endif -#if IS_ENABLED(CONFIG_MCF_EDMA) +#ifdef MCFEDMA_BASE &mcf_edma, #endif #ifdef MCFSDHC_BASE diff --git a/arch/m68k/coldfire/m5441x.c b/arch/m68k/coldfire/m5441x.c index 39855044090d..405e9d5c832c 100644 --- a/arch/m68k/coldfire/m5441x.c +++ b/arch/m68k/coldfire/m5441x.c @@ -181,7 +181,6 @@ static struct clk * const disable_clks[] __initconst = { &__clk_0_47, /* ssi.0 */ &__clk_0_49, /* rng */ &__clk_0_50, /* ssi.1 */ - &__clk_0_51, /* eSDHC */ &__clk_0_53, /* enet-fec */ &__clk_0_54, /* enet-fec */ &__clk_0_55, /* switch.0 */ diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index bc9952f8be66..114aaa3f955a 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -104,7 +104,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -204,7 +203,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -229,7 +227,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -435,6 +432,7 @@ CONFIG_FB_AMIGA_ECS=y CONFIG_FB_AMIGA_AGA=y CONFIG_FB_FM2=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION=y CONFIG_LOGO=y CONFIG_SOUND=m CONFIG_DMASOUND_PAULA=m @@ -500,7 +498,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -643,7 +640,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index a77269c6e5ba..30b9d932b930 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -100,7 +100,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -200,7 +199,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -225,7 +223,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -393,6 +390,7 @@ CONFIG_PTP_1588_CLOCK=m # CONFIG_HWMON is not set CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION=y CONFIG_LOGO=y # CONFIG_LOGO_LINUX_VGA16 is not set # CONFIG_LOGO_LINUX_CLUT224 is not set @@ -457,7 +455,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -599,7 +596,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 7a74efa6b9a1..51ff3180e69d 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -107,7 +107,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -207,7 +206,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -232,7 +230,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -478,7 +475,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -621,7 +617,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index a5323bf2eb33..7d95ca4366e4 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -97,7 +97,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -197,7 +196,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -222,7 +220,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -450,7 +447,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -592,7 +588,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 5e80aa0869d5..e306e3813607 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -99,7 +99,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -199,7 +198,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -224,7 +222,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -395,6 +392,7 @@ CONFIG_PTP_1588_CLOCK=m # CONFIG_HWMON is not set CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION=y CONFIG_LOGO=y # CONFIG_LOGO_LINUX_MONO is not set # CONFIG_LOGO_LINUX_VGA16 is not set @@ -459,7 +457,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -601,7 +598,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index e84326a3f62d..41316cf02441 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -98,7 +98,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -198,7 +197,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -223,7 +221,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -480,7 +477,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -623,7 +619,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 337552f43339..2fc3f0df6d43 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -118,7 +118,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -218,7 +217,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -243,7 +241,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -497,6 +494,7 @@ CONFIG_FB_ATARI=y CONFIG_FB_VALKYRIE=y CONFIG_FB_MAC=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION=y CONFIG_LOGO=y CONFIG_SOUND=m CONFIG_DMASOUND_ATARI=m @@ -565,7 +563,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -708,7 +705,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 7b688f7d272a..9603f4396469 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -96,7 +96,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -196,7 +195,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -221,7 +219,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -449,7 +446,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -591,7 +587,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 7c2cb31d63dd..c9cabd3344df 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -97,7 +97,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -197,7 +196,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -222,7 +220,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -450,7 +447,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -592,7 +588,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index ca43897af26d..5f994bf44fb8 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -98,7 +98,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -198,7 +197,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -223,7 +221,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -467,7 +464,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -610,7 +606,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index e3d515f37144..183e33f7d4a0 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -94,7 +94,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -194,7 +193,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -219,7 +217,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -388,9 +385,6 @@ CONFIG_NTP_PPS=y CONFIG_PPS_CLIENT_LDISC=m CONFIG_PTP_1588_CLOCK=m # CONFIG_HWMON is not set -CONFIG_FB=y -CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_LOGO=y CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m @@ -452,7 +446,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -593,7 +586,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index d601606c969b..8214263b9ab8 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -94,7 +94,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -194,7 +193,6 @@ CONFIG_IP_SET_LIST_SET=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m @@ -219,7 +217,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -387,9 +384,6 @@ CONFIG_NTP_PPS=y CONFIG_PPS_CLIENT_LDISC=m CONFIG_PTP_1588_CLOCK=m # CONFIG_HWMON is not set -CONFIG_FB=y -CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_LOGO=y CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m @@ -451,7 +445,6 @@ CONFIG_NFS_V4=m CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m # CONFIG_CIFS_STATS2 is not set # CONFIG_CIFS_DEBUG is not set @@ -593,7 +586,7 @@ CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m -CONFIG_TEST_HASH=m +CONFIG_TEST_SIPHASH=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 9c57b245dc12..267b02cc5655 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -13,7 +13,6 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/types.h> -#include <linux/genhd.h> #include <linux/blkdev.h> #include <linux/hdreg.h> #include <linux/slab.h> diff --git a/arch/m68k/hp300/config.c b/arch/m68k/hp300/config.c index ce1eb3d3d55d..2c92843397c3 100644 --- a/arch/m68k/hp300/config.c +++ b/arch/m68k/hp300/config.c @@ -22,6 +22,7 @@ #include <asm/blinken.h> #include <asm/io.h> /* readb() and writeb() */ #include <asm/hp300hw.h> +#include <asm/config.h> #include "time.h" diff --git a/arch/m68k/include/asm/cmpxchg.h b/arch/m68k/include/asm/cmpxchg.h index e8ca4b0ccefa..6cf464cdab06 100644 --- a/arch/m68k/include/asm/cmpxchg.h +++ b/arch/m68k/include/asm/cmpxchg.h @@ -4,8 +4,7 @@ #include <linux/irqflags.h> -struct __xchg_dummy { unsigned long a[100]; }; -#define __xg(x) ((volatile struct __xchg_dummy *)(x)) +#define __xg(type, x) ((volatile type *)(x)) extern unsigned long __invalid_xchg_size(unsigned long, volatile void *, int); @@ -50,7 +49,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz "1:\n\t" "casb %0,%1,%2\n\t" "jne 1b" - : "=&d" (x) : "d" (x), "m" (*__xg(ptr)) : "memory"); + : "=&d" (x) : "d" (x), "m" (*__xg(u8, ptr)) : "memory"); break; case 2: __asm__ __volatile__ @@ -58,7 +57,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz "1:\n\t" "casw %0,%1,%2\n\t" "jne 1b" - : "=&d" (x) : "d" (x), "m" (*__xg(ptr)) : "memory"); + : "=&d" (x) : "d" (x), "m" (*__xg(u16, ptr)) : "memory"); break; case 4: __asm__ __volatile__ @@ -66,7 +65,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz "1:\n\t" "casl %0,%1,%2\n\t" "jne 1b" - : "=&d" (x) : "d" (x), "m" (*__xg(ptr)) : "memory"); + : "=&d" (x) : "d" (x), "m" (*__xg(u32, ptr)) : "memory"); break; default: x = __invalid_xchg_size(x, ptr, size); diff --git a/arch/m68k/include/asm/config.h b/arch/m68k/include/asm/config.h new file mode 100644 index 000000000000..e73ffa23c4f5 --- /dev/null +++ b/arch/m68k/include/asm/config.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file contains prototypes provided by each m68k machine + * to parse bootinfo data structures and to configure the machine + */ + +#ifndef _M68K_CONFIG_H +#define _M68K_CONFIG_H + +extern int amiga_parse_bootinfo(const struct bi_record *record); +extern int apollo_parse_bootinfo(const struct bi_record *record); +extern int atari_parse_bootinfo(const struct bi_record *record); +extern int bvme6000_parse_bootinfo(const struct bi_record *record); +extern int hp300_parse_bootinfo(const struct bi_record *record); +extern int mac_parse_bootinfo(const struct bi_record *record); +extern int mvme147_parse_bootinfo(const struct bi_record *record); +extern int mvme16x_parse_bootinfo(const struct bi_record *record); +extern int q40_parse_bootinfo(const struct bi_record *record); + +extern void config_amiga(void); +extern void config_apollo(void); +extern void config_atari(void); +extern void config_bvme6000(void); +extern void config_hp300(void); +extern void config_mac(void); +extern void config_mvme147(void); +extern void config_mvme16x(void); +extern void config_q40(void); +extern void config_sun3(void); +extern void config_sun3x(void); + +#endif /* _M68K_CONFIG_H */ diff --git a/arch/m68k/include/asm/current.h b/arch/m68k/include/asm/current.h index 6390ef2f7f86..c117907e1276 100644 --- a/arch/m68k/include/asm/current.h +++ b/arch/m68k/include/asm/current.h @@ -24,6 +24,8 @@ static inline struct task_struct *get_current(void) #define current get_current() -#endif /* CONFNIG_MMU */ +#endif /* CONFIG_MMU */ + +register unsigned long current_stack_pointer __asm__("sp"); #endif /* !(_M68K_CURRENT_H) */ diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h index 6f2b87d7a50d..94f38d76e278 100644 --- a/arch/m68k/include/asm/mcf_pgtable.h +++ b/arch/m68k/include/asm/mcf_pgtable.h @@ -322,6 +322,7 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) (__pte((x).val)) +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) #define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index 022c3abc280d..7c9b56e2a750 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -147,6 +147,7 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) #define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE) #define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; }) +#define pmd_pfn(pmd) ((pmd_val(pmd) & _TABLE_MASK) >> PAGE_SHIFT) /* * m68k does not have huge pages (020/030 actually could), but generic code * expects pmd_page() to exists, only to then DCE it all. Provide a dummy to diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h index 5b24283a0a42..5e4e753f0d24 100644 --- a/arch/m68k/include/asm/sun3_pgtable.h +++ b/arch/m68k/include/asm/sun3_pgtable.h @@ -130,6 +130,7 @@ static inline void pte_clear (struct mm_struct *mm, unsigned long addr, pte_t *p ({ pte_t __pte; pte_val(__pte) = pfn | pgprot_val(pgprot); __pte; }) #define pte_page(pte) virt_to_page(__pte_page(pte)) +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd)) diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index ba670523885c..64914872a5c9 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h @@ -10,17 +10,7 @@ #include <linux/compiler.h> #include <linux/types.h> #include <asm/extable.h> - -/* We let the MMU do all checking */ -static inline int access_ok(const void __user *addr, - unsigned long size) -{ - /* - * XXX: for !CONFIG_CPU_HAS_ADDRESS_SPACES this really needs to check - * for TASK_SIZE! - */ - return 1; -} +#include <asm-generic/access_ok.h> /* * Not all varients of the 68k family support the notion of address spaces. @@ -390,8 +380,6 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) #define INLINE_COPY_FROM_USER #define INLINE_COPY_TO_USER -#define HAVE_GET_KERNEL_NOFAULT - #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ type *__gk_dst = (type *)(dst); \ diff --git a/arch/m68k/include/uapi/asm/signal.h b/arch/m68k/include/uapi/asm/signal.h index 4619291df601..80f520b9b10b 100644 --- a/arch/m68k/include/uapi/asm/signal.h +++ b/arch/m68k/include/uapi/asm/signal.h @@ -83,7 +83,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; #endif /* _UAPI_M68K_SIGNAL_H */ diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index 49e573b94326..8f94feed969c 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -16,7 +16,6 @@ #include <linux/interrupt.h> #include <linux/fs.h> #include <linux/console.h> -#include <linux/genhd.h> #include <linux/errno.h> #include <linux/string.h> #include <linux/init.h> @@ -47,6 +46,7 @@ #endif #include <asm/macintosh.h> #include <asm/natfeat.h> +#include <asm/config.h> #if !FPSTATESIZE || !NR_IRQS #warning No CPU/platform type selected, your kernel will not work! @@ -113,28 +113,6 @@ EXPORT_SYMBOL(isa_type); EXPORT_SYMBOL(isa_sex); #endif -extern int amiga_parse_bootinfo(const struct bi_record *); -extern int atari_parse_bootinfo(const struct bi_record *); -extern int mac_parse_bootinfo(const struct bi_record *); -extern int q40_parse_bootinfo(const struct bi_record *); -extern int bvme6000_parse_bootinfo(const struct bi_record *); -extern int mvme16x_parse_bootinfo(const struct bi_record *); -extern int mvme147_parse_bootinfo(const struct bi_record *); -extern int hp300_parse_bootinfo(const struct bi_record *); -extern int apollo_parse_bootinfo(const struct bi_record *); - -extern void config_amiga(void); -extern void config_atari(void); -extern void config_mac(void); -extern void config_sun3(void); -extern void config_apollo(void); -extern void config_mvme147(void); -extern void config_mvme16x(void); -extern void config_bvme6000(void); -extern void config_hp300(void); -extern void config_q40(void); -extern void config_sun3x(void); - #define MASK_256K 0xfffc0000 extern void paging_init(void); diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index 5d16f9b47aa9..65d124ec80bb 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -47,6 +47,7 @@ #include <asm/mac_via.h> #include <asm/mac_oss.h> #include <asm/mac_psc.h> +#include <asm/config.h> /* Mac bootinfo struct */ struct mac_booter_data mac_bi_data; diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 1493cf5eac1e..71aa9f6315dc 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -93,8 +93,6 @@ retry: vma = find_vma(mm, address); if (!vma) goto map_err; - if (vma->vm_flags & VM_IO) - goto acc_err; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c index dfd6202fd403..4e6218115f43 100644 --- a/arch/m68k/mvme147/config.c +++ b/arch/m68k/mvme147/config.c @@ -22,7 +22,6 @@ #include <linux/linkage.h> #include <linux/init.h> #include <linux/major.h> -#include <linux/genhd.h> #include <linux/rtc.h> #include <linux/interrupt.h> @@ -34,6 +33,7 @@ #include <asm/traps.h> #include <asm/machdep.h> #include <asm/mvme147hw.h> +#include <asm/config.h> static void mvme147_get_model(char *model); diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c index b4422c2dfbbf..f00c7aa058de 100644 --- a/arch/m68k/mvme16x/config.c +++ b/arch/m68k/mvme16x/config.c @@ -24,7 +24,6 @@ #include <linux/linkage.h> #include <linux/init.h> #include <linux/major.h> -#include <linux/genhd.h> #include <linux/rtc.h> #include <linux/interrupt.h> #include <linux/module.h> @@ -37,6 +36,7 @@ #include <asm/traps.h> #include <asm/machdep.h> #include <asm/mvme16xhw.h> +#include <asm/config.h> extern t_bdid mvme_bdid; diff --git a/arch/m68k/q40/config.c b/arch/m68k/q40/config.c index 5caf1e5be1c2..9237243077ce 100644 --- a/arch/m68k/q40/config.c +++ b/arch/m68k/q40/config.c @@ -34,6 +34,7 @@ #include <asm/traps.h> #include <asm/machdep.h> #include <asm/q40_master.h> +#include <asm/config.h> extern void q40_init_IRQ(void); static void q40_get_model(char *model); diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 59798e43cdb0..8cf429ad1c84 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -42,9 +42,10 @@ config MICROBLAZE select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE select SPARSE_IRQ - select SET_FS select ZONE_DMA select TRACE_IRQFLAGS_SUPPORT + select GENERIC_IRQ_MULTI_HANDLER + select HANDLE_DOMAIN_IRQ # Endianness selection choice diff --git a/arch/microblaze/include/asm/irq.h b/arch/microblaze/include/asm/irq.h index 0a28e80bbab0..cb6ab55d1d01 100644 --- a/arch/microblaze/include/asm/irq.h +++ b/arch/microblaze/include/asm/irq.h @@ -11,7 +11,4 @@ struct pt_regs; extern void do_IRQ(struct pt_regs *regs); -/* should be defined in each interrupt controller driver */ -extern unsigned int xintc_get_irq(void); - #endif /* _ASM_MICROBLAZE_IRQ_H */ diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index c136a01e467e..0c72646370e1 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -399,6 +399,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return ((unsigned long) (pmd_val(pmd) & PAGE_MASK)); } +/* returns pfn of the pmd entry*/ +#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) + /* returns struct *page of the pmd entry*/ #define pmd_page(pmd) (pfn_to_page(__pa(pmd_val(pmd)) >> PAGE_SHIFT)) diff --git a/arch/microblaze/include/asm/thread_info.h b/arch/microblaze/include/asm/thread_info.h index 44f5ca331862..a0ddd2a36fb9 100644 --- a/arch/microblaze/include/asm/thread_info.h +++ b/arch/microblaze/include/asm/thread_info.h @@ -56,17 +56,12 @@ struct cpu_context { __u32 fsr; }; -typedef struct { - unsigned long seg; -} mm_segment_t; - struct thread_info { struct task_struct *task; /* main task structure */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ __s32 preempt_count; /* 0 => preemptable,< 0 => BUG*/ - mm_segment_t addr_limit; /* thread address space */ struct cpu_context cpu_context; }; @@ -80,7 +75,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } /* how to get the thread information struct from C */ diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index d2a8ef9f8978..3aab2f17e046 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -15,48 +15,7 @@ #include <linux/pgtable.h> #include <asm/extable.h> #include <linux/string.h> - -/* - * On Microblaze the fs value is actually the top of the corresponding - * address space. - * - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - * - * For non-MMU arch like Microblaze, KERNEL_DS and USER_DS is equal. - */ -# define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -# define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFF) -# define USER_DS MAKE_MM_SEG(TASK_SIZE - 1) - -# define get_fs() (current_thread_info()->addr_limit) -# define set_fs(val) (current_thread_info()->addr_limit = (val)) -# define user_addr_max() get_fs().seg - -# define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -static inline int access_ok(const void __user *addr, unsigned long size) -{ - if (!size) - goto ok; - - if ((get_fs().seg < ((unsigned long)addr)) || - (get_fs().seg < ((unsigned long)addr + size - 1))) { - pr_devel("ACCESS fail at 0x%08x (size 0x%x), seg 0x%08x\n", - (__force u32)addr, (u32)size, - (u32)get_fs().seg); - return 0; - } -ok: - pr_devel("ACCESS OK at 0x%08x (size 0x%x), seg 0x%08x\n", - (__force u32)addr, (u32)size, - (u32)get_fs().seg); - return 1; -} +#include <asm-generic/access_ok.h> # define __FIXUP_SECTION ".section .fixup,\"ax\"\n" # define __EX_TABLE_SECTION ".section __ex_table,\"a\"\n" @@ -141,27 +100,27 @@ extern long __user_bad(void); #define __get_user(x, ptr) \ ({ \ - unsigned long __gu_val = 0; \ long __gu_err; \ switch (sizeof(*(ptr))) { \ case 1: \ - __get_user_asm("lbu", (ptr), __gu_val, __gu_err); \ + __get_user_asm("lbu", (ptr), x, __gu_err); \ break; \ case 2: \ - __get_user_asm("lhu", (ptr), __gu_val, __gu_err); \ + __get_user_asm("lhu", (ptr), x, __gu_err); \ break; \ case 4: \ - __get_user_asm("lw", (ptr), __gu_val, __gu_err); \ + __get_user_asm("lw", (ptr), x, __gu_err); \ break; \ - case 8: \ - __gu_err = __copy_from_user(&__gu_val, ptr, 8); \ - if (__gu_err) \ - __gu_err = -EFAULT; \ + case 8: { \ + __u64 __x = 0; \ + __gu_err = raw_copy_from_user(&__x, ptr, 8) ? \ + -EFAULT : 0; \ + (x) = (typeof(x))(typeof((x) - (x)))__x; \ break; \ + } \ default: \ /* __gu_val = 0; __gu_err = -EINVAL;*/ __gu_err = __user_bad();\ } \ - x = (__force __typeof__(*(ptr))) __gu_val; \ __gu_err; \ }) diff --git a/arch/microblaze/kernel/asm-offsets.c b/arch/microblaze/kernel/asm-offsets.c index b77dd188dec4..47ee409508b1 100644 --- a/arch/microblaze/kernel/asm-offsets.c +++ b/arch/microblaze/kernel/asm-offsets.c @@ -86,7 +86,6 @@ int main(int argc, char *argv[]) /* struct thread_info */ DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); - DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_CPU_CONTEXT, offsetof(struct thread_info, cpu_context)); DEFINE(TI_PREEMPT_COUNT, offsetof(struct thread_info, preempt_count)); BLANK(); diff --git a/arch/microblaze/kernel/irq.c b/arch/microblaze/kernel/irq.c index 903dad822fad..1f8cb4c4f74f 100644 --- a/arch/microblaze/kernel/irq.c +++ b/arch/microblaze/kernel/irq.c @@ -20,27 +20,13 @@ #include <linux/irqchip.h> #include <linux/of_irq.h> -static u32 concurrent_irq; - void __irq_entry do_IRQ(struct pt_regs *regs) { - unsigned int irq; struct pt_regs *old_regs = set_irq_regs(regs); trace_hardirqs_off(); irq_enter(); - irq = xintc_get_irq(); -next_irq: - BUG_ON(!irq); - generic_handle_irq(irq); - - irq = xintc_get_irq(); - if (irq != -1U) { - pr_debug("next irq: %d\n", irq); - ++concurrent_irq; - goto next_irq; - } - + handle_arch_irq(regs); irq_exit(); set_irq_regs(old_regs); trace_hardirqs_on(); diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 5e2b91c1e8ce..1b944d319d73 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -18,7 +18,6 @@ #include <linux/tick.h> #include <linux/bitops.h> #include <linux/ptrace.h> -#include <linux/uaccess.h> /* for USER_DS macros */ #include <asm/cacheflush.h> void show_regs(struct pt_regs *regs) diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts index 3e336b3dbb10..ab6e3dc0bc1d 100644 --- a/arch/mips/boot/dts/ingenic/ci20.dts +++ b/arch/mips/boot/dts/ingenic/ci20.dts @@ -83,6 +83,8 @@ label = "HDMI OUT"; type = "a"; + ddc-en-gpios = <&gpa 25 GPIO_ACTIVE_HIGH>; + port { hdmi_con: endpoint { remote-endpoint = <&dw_hdmi_out>; @@ -114,17 +116,6 @@ gpio = <&gpf 14 GPIO_ACTIVE_LOW>; enable-active-high; }; - - hdmi_power: fixedregulator@3 { - compatible = "regulator-fixed"; - - regulator-name = "hdmi_power"; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - - gpio = <&gpa 25 0>; - enable-active-high; - }; }; &ext { @@ -576,8 +567,6 @@ pinctrl-names = "default"; pinctrl-0 = <&pins_hdmi_ddc>; - hdmi-5v-supply = <&hdmi_power>; - ports { #address-cells = <1>; #size-cells = <0>; diff --git a/arch/mips/configs/cobalt_defconfig b/arch/mips/configs/cobalt_defconfig index c6a652ad34f7..e835730ea7fa 100644 --- a/arch/mips/configs/cobalt_defconfig +++ b/arch/mips/configs/cobalt_defconfig @@ -69,6 +69,5 @@ CONFIG_CONFIGFS_FS=y CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y CONFIG_LIBCRC32C=y diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index e2ed105f8c97..0021427a1bbe 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -159,7 +159,6 @@ CONFIG_NFS_V3_ACL=y CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y # CONFIG_RPCSEC_GSS_KRB5 is not set CONFIG_NLS_ISO8859_8=m diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 7e987d6f5e34..7a97a0818ce4 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -154,7 +154,6 @@ CONFIG_NFS_V3_ACL=y CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y # CONFIG_RPCSEC_GSS_KRB5 is not set CONFIG_NLS_ISO8859_8=m diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index 6df5f6f2ac8e..a0643363526d 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -154,7 +154,6 @@ CONFIG_NFS_V3_ACL=y CONFIG_NFS_SWAP=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y # CONFIG_RPCSEC_GSS_KRB5 is not set CONFIG_NLS_ISO8859_8=m diff --git a/arch/mips/configs/ip22_defconfig b/arch/mips/configs/ip22_defconfig index 21a1168ae301..70a4ba90f491 100644 --- a/arch/mips/configs/ip22_defconfig +++ b/arch/mips/configs/ip22_defconfig @@ -269,7 +269,6 @@ CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFS_V3_ACL=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NFSD_V3_ACL=y CONFIG_CIFS=m CONFIG_CIFS_UPCALL=y diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig index 1ae48f7d9ddd..74020aa3440b 100644 --- a/arch/mips/configs/ip32_defconfig +++ b/arch/mips/configs/ip32_defconfig @@ -112,7 +112,6 @@ CONFIG_CONFIGFS_FS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m CONFIG_NLS=y CONFIG_NLS_CODEPAGE_437=m diff --git a/arch/mips/configs/jazz_defconfig b/arch/mips/configs/jazz_defconfig index 8c223035921f..843f360da5f2 100644 --- a/arch/mips/configs/jazz_defconfig +++ b/arch/mips/configs/jazz_defconfig @@ -92,5 +92,4 @@ CONFIG_TMPFS=y CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index 3321bb576944..1c220d152a50 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -363,7 +363,6 @@ CONFIG_UFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index 009b30372226..b5ba08d7ab57 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -371,7 +371,6 @@ CONFIG_UFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index e214e136101c..8d58653f1b4e 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -370,7 +370,6 @@ CONFIG_UFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig index 3dc2da2bee0d..7d6f235e8ccb 100644 --- a/arch/mips/configs/rm200_defconfig +++ b/arch/mips/configs/rm200_defconfig @@ -354,7 +354,6 @@ CONFIG_SYSV_FS=m CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m CONFIG_CODA_FS=m CONFIG_AFS_FS=m diff --git a/arch/mips/configs/tb0219_defconfig b/arch/mips/configs/tb0219_defconfig index 6547f84750b5..c56d8ab14ba6 100644 --- a/arch/mips/configs/tb0219_defconfig +++ b/arch/mips/configs/tb0219_defconfig @@ -72,6 +72,5 @@ CONFIG_ROMFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="cca=3 mem=64M console=ttyVR0,115200 ip=any root=/dev/nfs" diff --git a/arch/mips/configs/tb0226_defconfig b/arch/mips/configs/tb0226_defconfig index 7e099f7c2286..6e1423428f02 100644 --- a/arch/mips/configs/tb0226_defconfig +++ b/arch/mips/configs/tb0226_defconfig @@ -67,6 +67,5 @@ CONFIG_ROMFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="cca=3 mem=32M console=ttyVR0,115200" diff --git a/arch/mips/configs/tb0287_defconfig b/arch/mips/configs/tb0287_defconfig index 0d881dd862c0..cf65a0879ece 100644 --- a/arch/mips/configs/tb0287_defconfig +++ b/arch/mips/configs/tb0287_defconfig @@ -77,7 +77,6 @@ CONFIG_ROMFS_FS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_FONTS=y CONFIG_FONT_8x8=y CONFIG_FONT_8x16=y diff --git a/arch/mips/configs/workpad_defconfig b/arch/mips/configs/workpad_defconfig index 4798dc86c9ce..7e16da0bde8c 100644 --- a/arch/mips/configs/workpad_defconfig +++ b/arch/mips/configs/workpad_defconfig @@ -63,6 +63,5 @@ CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_NFS_FS=m CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="console=ttyVR0,19200 ide0=0x170,0x376,49 mem=16M" diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 7b8037f25d9e..374c6322775d 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -86,6 +86,11 @@ extern void paging_init(void); */ #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return pmd_val(pmd) >> _PFN_SHIFT; +} + #ifndef CONFIG_MIPS_HUGE_TLB_SUPPORT #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ @@ -422,11 +427,6 @@ static inline int pmd_write(pmd_t pmd) return !!(pmd_val(pmd) & _PAGE_WRITE); } -static inline unsigned long pmd_pfn(pmd_t pmd) -{ - return pmd_val(pmd) >> _PFN_SHIFT; -} - static inline struct page *pmd_page(pmd_t pmd) { if (pmd_val(pmd) & _PAGE_HUGE) diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index f8f74f9f5883..c0cede273c7c 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -19,6 +19,7 @@ #ifdef CONFIG_32BIT #define __UA_LIMIT 0x80000000UL +#define TASK_SIZE_MAX KSEG0 #define __UA_ADDR ".word" #define __UA_LA "la" @@ -33,6 +34,7 @@ extern u64 __ua_limit; #define __UA_LIMIT __ua_limit +#define TASK_SIZE_MAX XKSSEG #define __UA_ADDR ".dword" #define __UA_LA "dla" @@ -42,50 +44,7 @@ extern u64 __ua_limit; #endif /* CONFIG_64BIT */ -/* - * Is a address valid? This does a straightforward calculation rather - * than tests. - * - * Address valid if: - * - "addr" doesn't have any high-bits set - * - AND "size" doesn't have any high-bits set - * - AND "addr+size" doesn't have any high-bits set - * - OR we are in kernel mode. - * - * __ua_size() is a trick to avoid runtime checking of positive constant - * sizes; for those we already know at compile time that the size is ok. - */ -#define __ua_size(size) \ - ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size)) - -/* - * access_ok: - Checks if a user space pointer is valid - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block may be valid, false (zero) - * if it is definitely invalid. - * - * Note that, depending on architecture, this function probably just - * checks that the pointer is in the user space range - after calling - * this function, memory access functions may still return -EFAULT. - */ - -static inline int __access_ok(const void __user *p, unsigned long size) -{ - unsigned long addr = (unsigned long)p; - unsigned long end = addr + size - !!size; - - return (__UA_LIMIT & (addr | end | __ua_size(size))) == 0; -} - -#define access_ok(addr, size) \ - likely(__access_ok((addr), (size))) +#include <asm-generic/access_ok.h> /* * put_user: - Write a simple value into user space. @@ -296,8 +255,6 @@ struct __large_struct { unsigned long buf[100]; }; (val) = __gu_tmp.t; \ } -#define HAVE_GET_KERNEL_NOFAULT - #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ int __gu_err; \ diff --git a/arch/mips/include/uapi/asm/shmbuf.h b/arch/mips/include/uapi/asm/shmbuf.h index 680bb95b2240..eb74d304b779 100644 --- a/arch/mips/include/uapi/asm/shmbuf.h +++ b/arch/mips/include/uapi/asm/shmbuf.h @@ -2,6 +2,9 @@ #ifndef _ASM_SHMBUF_H #define _ASM_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * The shmid64_ds structure for the MIPS architecture. * Note extra padding because this structure is passed back and forth @@ -16,7 +19,7 @@ #ifdef __mips64 struct shmid64_ds { struct ipc64_perm shm_perm; /* operation perms */ - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ long shm_atime; /* last attach time */ long shm_dtime; /* last detach time */ long shm_ctime; /* last change time */ @@ -29,7 +32,7 @@ struct shmid64_ds { #else struct shmid64_ds { struct ipc64_perm shm_perm; /* operation perms */ - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ unsigned long shm_atime; /* last attach time */ unsigned long shm_dtime; /* last detach time */ unsigned long shm_ctime; /* last change time */ diff --git a/arch/mips/include/uapi/asm/signal.h b/arch/mips/include/uapi/asm/signal.h index e6c78a15cb2f..94a00f82e373 100644 --- a/arch/mips/include/uapi/asm/signal.h +++ b/arch/mips/include/uapi/asm/signal.h @@ -100,7 +100,7 @@ struct sigaction { /* IRIX compatible stack_t */ typedef struct sigaltstack { void __user *ss_sp; - size_t ss_size; + __kernel_size_t ss_size; int ss_flags; } stack_t; diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index f979adfd4fc2..ef73ba1e0ec1 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -803,7 +803,7 @@ early_param("coherentio", setcoherentio); static int __init setnocoherentio(char *str) { - dma_default_coherent = true; + dma_default_coherent = false; pr_info("Software DMA cache coherency (command line)\n"); return 0; } diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index d542fb7af3ba..1986d1309410 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -351,6 +351,9 @@ asmlinkage void start_secondary(void) cpu = smp_processor_id(); cpu_data[cpu].udelay_val = loops_per_jiffy; + set_cpu_sibling_map(cpu); + set_cpu_core_map(cpu); + cpumask_set_cpu(cpu, &cpu_coherent_mask); notify_cpu_starting(cpu); @@ -362,9 +365,6 @@ asmlinkage void start_secondary(void) /* The CPU is running and counters synchronised, now mark it online */ set_cpu_online(cpu, true); - set_cpu_sibling_map(cpu); - set_cpu_core_map(cpu); - calculate_cpu_foreign_map(); /* diff --git a/arch/mips/kernel/topology.c b/arch/mips/kernel/topology.c index 08ad6371fbe0..9429d85a4703 100644 --- a/arch/mips/kernel/topology.c +++ b/arch/mips/kernel/topology.c @@ -12,11 +12,6 @@ static int __init topology_init(void) { int i, ret; -#ifdef CONFIG_NUMA - for_each_online_node(i) - register_one_node(i); -#endif /* CONFIG_NUMA */ - for_each_present_cpu(i) { struct cpu *c = &per_cpu(cpu_devices, i); diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index df4b708c04a9..7b5aba5df02e 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -1480,6 +1480,23 @@ asmlinkage void do_ade(struct pt_regs *regs) prev_state = exception_enter(); perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->cp0_badvaddr); + +#ifdef CONFIG_64BIT + /* + * check, if we are hitting space between CPU implemented maximum + * virtual user address and 64bit maximum virtual user address + * and do exception handling to get EFAULTs for get_user/put_user + */ + if ((regs->cp0_badvaddr >= (1UL << cpu_vmbits)) && + (regs->cp0_badvaddr < XKSSEG)) { + if (fixup_exception(regs)) { + current->thread.cp0_baduaddr = regs->cp0_badvaddr; + return; + } + goto sigbus; + } +#endif + /* * Did we catch a fault trying to load an instruction? */ diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index d6efffd4dd20..fb0565bc34fd 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -22,7 +22,9 @@ #include "common.h" -static void *detect_magic __initdata = detect_memory_region; +#define MT7621_MEM_TEST_PATTERN 0xaa5555aa + +static u32 detect_magic __initdata; int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) { @@ -58,24 +60,32 @@ phys_addr_t mips_cpc_default_phys_base(void) panic("Cannot detect cpc address"); } +static bool __init mt7621_addr_wraparound_test(phys_addr_t size) +{ + void *dm = (void *)KSEG1ADDR(&detect_magic); + + if (CPHYSADDR(dm + size) >= MT7621_LOWMEM_MAX_SIZE) + return true; + __raw_writel(MT7621_MEM_TEST_PATTERN, dm); + if (__raw_readl(dm) != __raw_readl(dm + size)) + return false; + __raw_writel(~MT7621_MEM_TEST_PATTERN, dm); + return __raw_readl(dm) == __raw_readl(dm + size); +} + static void __init mt7621_memory_detect(void) { - void *dm = &detect_magic; phys_addr_t size; - for (size = 32 * SZ_1M; size < 256 * SZ_1M; size <<= 1) { - if (!__builtin_memcmp(dm, dm + size, sizeof(detect_magic))) - break; + for (size = 32 * SZ_1M; size <= 256 * SZ_1M; size <<= 1) { + if (mt7621_addr_wraparound_test(size)) { + memblock_add(MT7621_LOWMEM_BASE, size); + return; + } } - if ((size == 256 * SZ_1M) && - (CPHYSADDR(dm + size) < MT7621_LOWMEM_MAX_SIZE) && - __builtin_memcmp(dm, dm + size, sizeof(detect_magic))) { - memblock_add(MT7621_LOWMEM_BASE, MT7621_LOWMEM_MAX_SIZE); - memblock_add(MT7621_HIGHMEM_BASE, MT7621_HIGHMEM_SIZE); - } else { - memblock_add(MT7621_LOWMEM_BASE, size); - } + memblock_add(MT7621_LOWMEM_BASE, MT7621_LOWMEM_MAX_SIZE); + memblock_add(MT7621_HIGHMEM_BASE, MT7621_HIGHMEM_SIZE); } void __init ralink_of_remap(void) diff --git a/arch/nds32/Kbuild b/arch/nds32/Kbuild deleted file mode 100644 index 4e39f7abdeb6..000000000000 --- a/arch/nds32/Kbuild +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -# for cleaning -subdir- += boot diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig deleted file mode 100644 index 4d1421b18734..000000000000 --- a/arch/nds32/Kconfig +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# For a description of the syntax of this configuration file, -# see Documentation/kbuild/kconfig-language.rst. -# - -config NDS32 - def_bool y - select ARCH_32BIT_OFF_T - select ARCH_HAS_DMA_PREP_COHERENT - select ARCH_HAS_SYNC_DMA_FOR_CPU - select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_WANT_FRAME_POINTERS if FTRACE - select CLKSRC_MMIO - select CLONE_BACKWARDS - select COMMON_CLK - select DMA_DIRECT_REMAP - select GENERIC_ATOMIC64 - select GENERIC_CPU_DEVICES - select GENERIC_IRQ_CHIP - select GENERIC_IRQ_SHOW - select GENERIC_IOREMAP - select GENERIC_LIB_ASHLDI3 - select GENERIC_LIB_ASHRDI3 - select GENERIC_LIB_CMPDI2 - select GENERIC_LIB_LSHRDI3 - select GENERIC_LIB_MULDI3 - select GENERIC_LIB_UCMPDI2 - select GENERIC_TIME_VSYSCALL - select HAVE_ARCH_TRACEHOOK - select HAVE_DEBUG_KMEMLEAK - select HAVE_EXIT_THREAD - select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_PERF_EVENTS - select IRQ_DOMAIN - select LOCKDEP_SUPPORT - select MODULES_USE_ELF_RELA - select OF - select OF_EARLY_FLATTREE - select NO_IOPORT_MAP - select RTC_LIB - select THREAD_INFO_IN_TASK - select HAVE_FUNCTION_TRACER - select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FTRACE_MCOUNT_RECORD - select HAVE_DYNAMIC_FTRACE - select SET_FS - select TRACE_IRQFLAGS_SUPPORT - help - Andes(nds32) Linux support. - -config GENERIC_CALIBRATE_DELAY - def_bool y - -config GENERIC_CSUM - def_bool y - -config GENERIC_HWEIGHT - def_bool y - -config GENERIC_LOCKBREAK - def_bool y - depends on PREEMPTION - -config STACKTRACE_SUPPORT - def_bool y - -config FIX_EARLYCON_MEM - def_bool y - -config PGTABLE_LEVELS - default 2 - -menu "System Type" -source "arch/nds32/Kconfig.cpu" -config NR_CPUS - int - default 1 - -config MMU - def_bool y - -config NDS32_BUILTIN_DTB - string "Builtin DTB" - default "" - help - User can use it to specify the dts of the SoC -endmenu - -menu "Kernel Features" -source "kernel/Kconfig.hz" -endmenu - -menu "Power management options" -config SYS_SUPPORTS_APM_EMULATION - bool - -config ARCH_SUSPEND_POSSIBLE - def_bool y - -source "kernel/power/Kconfig" -endmenu diff --git a/arch/nds32/Kconfig.cpu b/arch/nds32/Kconfig.cpu deleted file mode 100644 index c10759952485..000000000000 --- a/arch/nds32/Kconfig.cpu +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -comment "Processor Features" - -config CPU_BIG_ENDIAN - def_bool !CPU_LITTLE_ENDIAN - -config CPU_LITTLE_ENDIAN - bool "Little endian" - default y - -config FPU - bool "FPU support" - default n - help - If FPU ISA is used in user space, this configuration shall be Y to - enable required support in kernel such as fpu context switch and - fpu exception handler. - - If no FPU ISA is used in user space, say N. - -config LAZY_FPU - bool "lazy FPU support" - depends on FPU - default y - help - Say Y here to enable the lazy FPU scheme. The lazy FPU scheme can - enhance system performance by reducing the context switch - frequency of the FPU register. - - For normal case, say Y. - -config SUPPORT_DENORMAL_ARITHMETIC - bool "Denormal arithmetic support" - depends on FPU - default n - help - Say Y here to enable arithmetic of denormalized number. Enabling - this feature can enhance the precision for tininess number. - However, performance loss in float point calculations is - possibly significant due to additional FPU exception. - - If the calculated tolerance for tininess number is not critical, - say N to prevent performance loss. - -config HWZOL - bool "hardware zero overhead loop support" - depends on CPU_D10 || CPU_D15 - default n - help - A set of Zero-Overhead Loop mechanism is provided to reduce the - instruction fetch and execution overhead of loop-control instructions. - It will save 3 registers($LB, $LC, $LE) for context saving if say Y. - You don't need to save these registers if you can make sure your user - program doesn't use these registers. - - If unsure, say N. - -config CPU_CACHE_ALIASING - bool "Aliasing cache" - depends on CPU_N10 || CPU_D10 || CPU_N13 || CPU_V3 - default y - help - If this CPU is using VIPT data cache and its cache way size is larger - than page size, say Y. If it is using PIPT data cache, say N. - - If unsure, say Y. - -choice - prompt "minimum CPU type" - default CPU_V3 - help - The data cache of N15/D15 is implemented as PIPT and it will not cause - the cache aliasing issue. The rest cpus(N13, N10 and D10) are - implemented as VIPT data cache. It may cause the cache aliasing issue - if its cache way size is larger than page size. You can specify the - CPU type directly or choose CPU_V3 if unsure. - - A kernel built for N10 is able to run on N15, D15, N13, N10 or D10. - A kernel built for N15 is able to run on N15 or D15. - A kernel built for D10 is able to run on D10 or D15. - A kernel built for D15 is able to run on D15. - A kernel built for N13 is able to run on N15, N13 or D15. - -config CPU_N15 - bool "AndesCore N15" -config CPU_N13 - bool "AndesCore N13" - select CPU_CACHE_ALIASING if ANDES_PAGE_SIZE_4KB -config CPU_N10 - bool "AndesCore N10" - select CPU_CACHE_ALIASING -config CPU_D15 - bool "AndesCore D15" -config CPU_D10 - bool "AndesCore D10" - select CPU_CACHE_ALIASING -config CPU_V3 - bool "AndesCore v3 compatible" - select CPU_CACHE_ALIASING -endchoice -choice - prompt "Paging -- page size " - default ANDES_PAGE_SIZE_4KB -config ANDES_PAGE_SIZE_4KB - bool "use 4KB page size" -config ANDES_PAGE_SIZE_8KB - bool "use 8KB page size" -endchoice - -config CPU_ICACHE_DISABLE - bool "Disable I-Cache" - help - Say Y here to disable the processor instruction cache. Unless - you have a reason not to or are unsure, say N. - -config CPU_DCACHE_DISABLE - bool "Disable D-Cache" - help - Say Y here to disable the processor data cache. Unless - you have a reason not to or are unsure, say N. - -config CPU_DCACHE_WRITETHROUGH - bool "Force write through D-cache" - depends on !CPU_DCACHE_DISABLE - help - Say Y here to use the data cache in writethrough mode. Unless you - specifically require this or are unsure, say N. - -config WBNA - bool "WBNA" - default n - help - Say Y here to enable write-back memory with no-write-allocation policy. - -config ALIGNMENT_TRAP - bool "Kernel support unaligned access handling by sw" - depends on PROC_FS - default n - help - Andes processors cannot load/store information which is not - naturally aligned on the bus, i.e., a 4 byte load must start at an - address divisible by 4. On 32-bit Andes processors, these non-aligned - load/store instructions will be emulated in software if you say Y - here, which has a severe performance impact. With an IP-only - configuration it is safe to say N, otherwise say Y. - -config HW_SUPPORT_UNALIGNMENT_ACCESS - bool "Kernel support unaligned access handling by hw" - depends on !ALIGNMENT_TRAP - default n - help - Andes processors load/store world/half-word instructions can access - unaligned memory locations without generating the Data Alignment - Check exceptions. With an IP-only configuration it is safe to say N, - otherwise say Y. - -config HIGHMEM - bool "High Memory Support" - depends on MMU && !CPU_CACHE_ALIASING - select KMAP_LOCAL - help - The address space of Andes processors is only 4 Gigabytes large - and it has to accommodate user address space, kernel address - space as well as some memory mapped IO. That means that, if you - have a large amount of physical memory and/or IO, not all of the - memory can be "permanently mapped" by the kernel. The physical - memory that is not permanently mapped is called "high memory". - - Depending on the selected kernel/user memory split, minimum - vmalloc space and actual amount of RAM, you may not need this - option which should result in a slightly faster kernel. - - If unsure, say N. - -config CACHE_L2 - bool "Support L2 cache" - default y - help - Say Y here to enable L2 cache if your SoC are integrated with L2CC. - If unsure, say N. - -config HW_PRE - bool "Enable hardware prefetcher" - default y - help - Say Y here to enable hardware prefetcher feature. - Only when CPU_VER.REV >= 0x09 can support. - -menu "Memory configuration" - -choice - prompt "Memory split" - depends on MMU - default VMSPLIT_3G_OPT - help - Select the desired split between kernel and user memory. - - If you are not absolutely sure what you are doing, leave this - option alone! - - config VMSPLIT_3G - bool "3G/1G user/kernel split" - config VMSPLIT_3G_OPT - bool "3G/1G user/kernel split (for full 1G low memory)" - config VMSPLIT_2G - bool "2G/2G user/kernel split" - config VMSPLIT_1G - bool "1G/3G user/kernel split" -endchoice - -config PAGE_OFFSET - hex - default 0x40000000 if VMSPLIT_1G - default 0x80000000 if VMSPLIT_2G - default 0xB0000000 if VMSPLIT_3G_OPT - default 0xC0000000 - -endmenu diff --git a/arch/nds32/Kconfig.debug b/arch/nds32/Kconfig.debug deleted file mode 100644 index 295942fe3fd5..000000000000 --- a/arch/nds32/Kconfig.debug +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# dummy file, do not delete diff --git a/arch/nds32/Makefile b/arch/nds32/Makefile deleted file mode 100644 index b33d5d81b6ae..000000000000 --- a/arch/nds32/Makefile +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -LDFLAGS_vmlinux := --no-undefined -X -OBJCOPYFLAGS := -O binary -R .note -R .note.gnu.build-id -R .comment -S - -ifdef CONFIG_FUNCTION_TRACER -arch-y += -malways-save-lp -mno-relax -endif - -# Avoid generating FPU instructions -arch-y += -mno-ext-fpu-sp -mno-ext-fpu-dp -mfloat-abi=soft - -# Enable <nds32_intrinsic.h> -KBUILD_CFLAGS += -isystem $(shell $(CC) -print-file-name=include) -KBUILD_CFLAGS += $(call cc-option, -mno-sched-prolog-epilog) -KBUILD_CFLAGS += -mcmodel=large - -KBUILD_CFLAGS +=$(arch-y) $(tune-y) -KBUILD_AFLAGS +=$(arch-y) $(tune-y) - -#Default value -head-y := arch/nds32/kernel/head.o -textaddr-y := $(CONFIG_PAGE_OFFSET)+0xc000 - -TEXTADDR := $(textaddr-y) - -export TEXTADDR - - -# If we have a machine-specific directory, then include it in the build. -core-y += arch/nds32/kernel/ arch/nds32/mm/ -core-$(CONFIG_FPU) += arch/nds32/math-emu/ -libs-y += arch/nds32/lib/ - -ifdef CONFIG_CPU_LITTLE_ENDIAN -KBUILD_CFLAGS += $(call cc-option, -EL) -KBUILD_AFLAGS += $(call cc-option, -EL) -KBUILD_LDFLAGS += $(call cc-option, -EL) -CHECKFLAGS += -D__NDS32_EL__ -else -KBUILD_CFLAGS += $(call cc-option, -EB) -KBUILD_AFLAGS += $(call cc-option, -EB) -KBUILD_LDFLAGS += $(call cc-option, -EB) -CHECKFLAGS += -D__NDS32_EB__ -endif - -boot := arch/nds32/boot -core-y += $(boot)/dts/ - -Image: vmlinux - $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ - - -PHONY += vdso_install -vdso_install: - $(Q)$(MAKE) $(build)=arch/nds32/kernel/vdso $@ - -prepare: vdso_prepare -vdso_prepare: prepare0 - $(Q)$(MAKE) $(build)=arch/nds32/kernel/vdso include/generated/vdso-offsets.h - -define archhelp - echo ' Image - kernel image (arch/$(ARCH)/boot/Image)' -endef diff --git a/arch/nds32/boot/.gitignore b/arch/nds32/boot/.gitignore deleted file mode 100644 index 9182a3a1ea0a..000000000000 --- a/arch/nds32/boot/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -/Image diff --git a/arch/nds32/boot/Makefile b/arch/nds32/boot/Makefile deleted file mode 100644 index c4cc0c2689f7..000000000000 --- a/arch/nds32/boot/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -targets := Image Image.gz - -$(obj)/Image: vmlinux FORCE - $(call if_changed,objcopy) - -$(obj)/Image.gz: $(obj)/Image FORCE - $(call if_changed,gzip) - -install: $(obj)/Image - $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ - $(obj)/Image System.map "$(INSTALL_PATH)" - -zinstall: $(obj)/Image.gz - $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ - $(obj)/Image.gz System.map "$(INSTALL_PATH)" diff --git a/arch/nds32/boot/dts/Makefile b/arch/nds32/boot/dts/Makefile deleted file mode 100644 index 4fc69562eae8..000000000000 --- a/arch/nds32/boot/dts/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_OF) += $(addsuffix .dtb.o, $(CONFIG_NDS32_BUILTIN_DTB)) diff --git a/arch/nds32/boot/dts/ae3xx.dts b/arch/nds32/boot/dts/ae3xx.dts deleted file mode 100644 index 16a9f54a805e..000000000000 --- a/arch/nds32/boot/dts/ae3xx.dts +++ /dev/null @@ -1,90 +0,0 @@ -/dts-v1/; -/ { - compatible = "andestech,ae3xx"; - #address-cells = <1>; - #size-cells = <1>; - interrupt-parent = <&intc>; - - chosen { - stdout-path = &serial0; - }; - - memory@0 { - device_type = "memory"; - reg = <0x00000000 0x40000000>; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - cpu@0 { - device_type = "cpu"; - compatible = "andestech,n13", "andestech,nds32v3"; - reg = <0>; - clock-frequency = <60000000>; - next-level-cache = <&L2>; - }; - }; - - intc: interrupt-controller { - compatible = "andestech,ativic32"; - #interrupt-cells = <1>; - interrupt-controller; - }; - - clock: clk { - #clock-cells = <0>; - compatible = "fixed-clock"; - clock-frequency = <30000000>; - }; - - apb { - compatible = "simple-bus"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - serial0: serial@f0300000 { - compatible = "andestech,uart16550", "ns16550a"; - reg = <0xf0300000 0x1000>; - interrupts = <8>; - clock-frequency = <14745600>; - reg-shift = <2>; - reg-offset = <32>; - no-loopback-test = <1>; - }; - - timer0: timer@f0400000 { - compatible = "andestech,atcpit100"; - reg = <0xf0400000 0x1000>; - interrupts = <2>; - clocks = <&clock>; - clock-names = "PCLK"; - }; - }; - - ahb { - compatible = "simple-bus"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - L2: cache-controller@e0500000 { - compatible = "andestech,atl2c"; - reg = <0xe0500000 0x1000>; - cache-unified; - cache-level = <2>; - }; - - mac0: ethernet@e0100000 { - compatible = "andestech,atmac100"; - reg = <0xe0100000 0x1000>; - interrupts = <18>; - }; - }; - - pmu { - compatible = "andestech,nds32v3-pmu"; - interrupts= <13>; - }; -}; diff --git a/arch/nds32/configs/defconfig b/arch/nds32/configs/defconfig deleted file mode 100644 index f9a89cf00aa6..000000000000 --- a/arch/nds32/configs/defconfig +++ /dev/null @@ -1,104 +0,0 @@ -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_BSD_PROCESS_ACCT_V3=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_USER_NS=y -CONFIG_RELAY=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_KALLSYMS_ALL=y -CONFIG_PROFILING=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -# CONFIG_BLK_DEV_BSG is not set -# CONFIG_CACHE_L2 is not set -CONFIG_PREEMPT=y -# CONFIG_COMPACTION is not set -CONFIG_HZ_100=y -# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_NET_KEY=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_DIAG is not set -# CONFIG_IPV6 is not set -# CONFIG_BLK_DEV is not set -CONFIG_NETDEVICES=y -# CONFIG_NET_CADENCE is not set -# CONFIG_NET_VENDOR_BROADCOM is not set -CONFIG_FTMAC100=y -# CONFIG_NET_VENDOR_INTEL is not set -# CONFIG_NET_VENDOR_MARVELL is not set -# CONFIG_NET_VENDOR_MICREL is not set -# CONFIG_NET_VENDOR_NATSEMI is not set -# CONFIG_NET_VENDOR_SEEQ is not set -# CONFIG_NET_VENDOR_STMICRO is not set -# CONFIG_NET_VENDOR_WIZNET is not set -CONFIG_INPUT_EVDEV=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_INPUT_TOUCHSCREEN=y -# CONFIG_SERIO is not set -CONFIG_VT_HW_CONSOLE_BINDING=y -CONFIG_SERIAL_8250=y -# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=3 -CONFIG_SERIAL_8250_RUNTIME_UARTS=3 -CONFIG_SERIAL_OF_PLATFORM=y -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_HID_A4TECH is not set -# CONFIG_HID_APPLE is not set -# CONFIG_HID_BELKIN is not set -# CONFIG_HID_CHERRY is not set -# CONFIG_HID_CHICONY is not set -# CONFIG_HID_CYPRESS is not set -# CONFIG_HID_EZKEY is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_KENSINGTON is not set -# CONFIG_HID_LOGITECH is not set -# CONFIG_HID_MICROSOFT is not set -# CONFIG_HID_MONTEREY is not set -# CONFIG_USB_SUPPORT is not set -CONFIG_GENERIC_PHY=y -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_EXT4_FS_SECURITY=y -CONFIG_FS_ENCRYPTION=y -CONFIG_FUSE_FS=y -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_TMPFS=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_CONFIGFS_FS=y -CONFIG_NFS_FS=y -CONFIG_NFS_V3_ACL=y -CONFIG_NFS_V4=y -CONFIG_NFS_V4_1=y -CONFIG_NFS_USE_LEGACY_DNS=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=y -CONFIG_DEBUG_INFO=y -CONFIG_DEBUG_INFO_DWARF4=y -CONFIG_GDB_SCRIPTS=y -CONFIG_READABLE_ASM=y -CONFIG_HEADERS_INSTALL=y -CONFIG_HEADERS_CHECK=y -CONFIG_DEBUG_SECTION_MISMATCH=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_KERNEL=y -CONFIG_PANIC_ON_OOPS=y -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_PREEMPT is not set -CONFIG_STACKTRACE=y -CONFIG_RCU_CPU_STALL_TIMEOUT=300 -# CONFIG_CRYPTO_HW is not set diff --git a/arch/nds32/include/asm/Kbuild b/arch/nds32/include/asm/Kbuild deleted file mode 100644 index 82a4453c9c2d..000000000000 --- a/arch/nds32/include/asm/Kbuild +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -generic-y += asm-offsets.h -generic-y += cmpxchg.h -generic-y += export.h -generic-y += gpio.h -generic-y += kvm_para.h -generic-y += parport.h -generic-y += user.h diff --git a/arch/nds32/include/asm/assembler.h b/arch/nds32/include/asm/assembler.h deleted file mode 100644 index 5e7c56926049..000000000000 --- a/arch/nds32/include/asm/assembler.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_ASSEMBLER_H__ -#define __NDS32_ASSEMBLER_H__ - -.macro gie_disable - setgie.d - dsb -.endm - -.macro gie_enable - setgie.e - dsb -.endm - -.macro gie_save oldpsw - mfsr \oldpsw, $ir0 - setgie.d - dsb -.endm - -.macro gie_restore oldpsw - andi \oldpsw, \oldpsw, #0x1 - beqz \oldpsw, 7001f - setgie.e - dsb -7001: -.endm - - -#define USER(insn, reg, addr, opr) \ -9999: insn reg, addr, opr; \ - .section __ex_table,"a"; \ - .align 3; \ - .long 9999b, 9001f; \ - .previous - -#endif /* __NDS32_ASSEMBLER_H__ */ diff --git a/arch/nds32/include/asm/barrier.h b/arch/nds32/include/asm/barrier.h deleted file mode 100644 index 16413172fd50..000000000000 --- a/arch/nds32/include/asm/barrier.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_ASM_BARRIER_H -#define __NDS32_ASM_BARRIER_H - -#ifndef __ASSEMBLY__ -#define mb() asm volatile("msync all":::"memory") -#define rmb() asm volatile("msync all":::"memory") -#define wmb() asm volatile("msync store":::"memory") -#include <asm-generic/barrier.h> - -#endif /* __ASSEMBLY__ */ - -#endif /* __NDS32_ASM_BARRIER_H */ diff --git a/arch/nds32/include/asm/bitfield.h b/arch/nds32/include/asm/bitfield.h deleted file mode 100644 index b02a58e71f80..000000000000 --- a/arch/nds32/include/asm/bitfield.h +++ /dev/null @@ -1,985 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_BITFIELD_H__ -#define __NDS32_BITFIELD_H__ -/****************************************************************************** - * cr0: CPU_VER (CPU Version Register) - *****************************************************************************/ -#define CPU_VER_offCFGID 0 /* Minor configuration */ -#define CPU_VER_offREV 16 /* Revision of the CPU version */ -#define CPU_VER_offCPUID 24 /* Major CPU versions */ - -#define CPU_VER_mskCFGID ( 0xFFFF << CPU_VER_offCFGID ) -#define CPU_VER_mskREV ( 0xFF << CPU_VER_offREV ) -#define CPU_VER_mskCPUID ( 0xFF << CPU_VER_offCPUID ) - -/****************************************************************************** - * cr1: ICM_CFG (Instruction Cache/Memory Configuration Register) - *****************************************************************************/ -#define ICM_CFG_offISET 0 /* I-cache sets (# of cache lines) per way */ -#define ICM_CFG_offIWAY 3 /* I-cache ways */ -#define ICM_CFG_offISZ 6 /* I-cache line size */ -#define ICM_CFG_offILCK 9 /* I-cache locking support */ -#define ICM_CFG_offILMB 10 /* On-chip ILM banks */ -#define ICM_CFG_offBSAV 13 /* ILM base register alignment version */ -/* bit 15:31 reserved */ - -#define ICM_CFG_mskISET ( 0x7 << ICM_CFG_offISET ) -#define ICM_CFG_mskIWAY ( 0x7 << ICM_CFG_offIWAY ) -#define ICM_CFG_mskISZ ( 0x7 << ICM_CFG_offISZ ) -#define ICM_CFG_mskILCK ( 0x1 << ICM_CFG_offILCK ) -#define ICM_CFG_mskILMB ( 0x7 << ICM_CFG_offILMB ) -#define ICM_CFG_mskBSAV ( 0x3 << ICM_CFG_offBSAV ) - -/****************************************************************************** - * cr2: DCM_CFG (Data Cache/Memory Configuration Register) - *****************************************************************************/ -#define DCM_CFG_offDSET 0 /* D-cache sets (# of cache lines) per way */ -#define DCM_CFG_offDWAY 3 /* D-cache ways */ -#define DCM_CFG_offDSZ 6 /* D-cache line size */ -#define DCM_CFG_offDLCK 9 /* D-cache locking support */ -#define DCM_CFG_offDLMB 10 /* On-chip DLM banks */ -#define DCM_CFG_offBSAV 13 /* DLM base register alignment version */ -/* bit 15:31 reserved */ - -#define DCM_CFG_mskDSET ( 0x7 << DCM_CFG_offDSET ) -#define DCM_CFG_mskDWAY ( 0x7 << DCM_CFG_offDWAY ) -#define DCM_CFG_mskDSZ ( 0x7 << DCM_CFG_offDSZ ) -#define DCM_CFG_mskDLCK ( 0x1 << DCM_CFG_offDLCK ) -#define DCM_CFG_mskDLMB ( 0x7 << DCM_CFG_offDLMB ) -#define DCM_CFG_mskBSAV ( 0x3 << DCM_CFG_offBSAV ) - -/****************************************************************************** - * cr3: MMU_CFG (MMU Configuration Register) - *****************************************************************************/ -#define MMU_CFG_offMMPS 0 /* Memory management protection scheme */ -#define MMU_CFG_offMMPV 2 /* Memory management protection version number */ -#define MMU_CFG_offFATB 7 /* Fully-associative or non-fully-associative TLB */ - -#define MMU_CFG_offTBW 8 /* TLB ways(non-associative) TBS */ -#define MMU_CFG_offTBS 11 /* TLB sets per way(non-associative) TBS */ -/* bit 14:14 reserved */ - -#define MMU_CFG_offEP8MIN4 15 /* 8KB page supported while minimum page is 4KB */ -#define MMU_CFG_offfEPSZ 16 /* Extra page size supported */ -#define MMU_CFG_offTLBLCK 24 /* TLB locking support */ -#define MMU_CFG_offHPTWK 25 /* Hardware Page Table Walker implemented */ -#define MMU_CFG_offDE 26 /* Default endian */ -#define MMU_CFG_offNTPT 27 /* Partitions for non-translated attributes */ -#define MMU_CFG_offIVTB 28 /* Invisible TLB */ -#define MMU_CFG_offVLPT 29 /* VLPT for fast TLB fill handling implemented */ -#define MMU_CFG_offNTME 30 /* Non-translated VA to PA mapping */ -/* bit 31 reserved */ - -#define MMU_CFG_mskMMPS ( 0x3 << MMU_CFG_offMMPS ) -#define MMU_CFG_mskMMPV ( 0x1F << MMU_CFG_offMMPV ) -#define MMU_CFG_mskFATB ( 0x1 << MMU_CFG_offFATB ) -#define MMU_CFG_mskTBW ( 0x7 << MMU_CFG_offTBW ) -#define MMU_CFG_mskTBS ( 0x7 << MMU_CFG_offTBS ) -#define MMU_CFG_mskEP8MIN4 ( 0x1 << MMU_CFG_offEP8MIN4 ) -#define MMU_CFG_mskfEPSZ ( 0xFF << MMU_CFG_offfEPSZ ) -#define MMU_CFG_mskTLBLCK ( 0x1 << MMU_CFG_offTLBLCK ) -#define MMU_CFG_mskHPTWK ( 0x1 << MMU_CFG_offHPTWK ) -#define MMU_CFG_mskDE ( 0x1 << MMU_CFG_offDE ) -#define MMU_CFG_mskNTPT ( 0x1 << MMU_CFG_offNTPT ) -#define MMU_CFG_mskIVTB ( 0x1 << MMU_CFG_offIVTB ) -#define MMU_CFG_mskVLPT ( 0x1 << MMU_CFG_offVLPT ) -#define MMU_CFG_mskNTME ( 0x1 << MMU_CFG_offNTME ) - -/****************************************************************************** - * cr4: MSC_CFG (Misc Configuration Register) - *****************************************************************************/ -#define MSC_CFG_offEDM 0 -#define MSC_CFG_offLMDMA 1 -#define MSC_CFG_offPFM 2 -#define MSC_CFG_offHSMP 3 -#define MSC_CFG_offTRACE 4 -#define MSC_CFG_offDIV 5 -#define MSC_CFG_offMAC 6 -#define MSC_CFG_offAUDIO 7 -#define MSC_CFG_offL2C 9 -#define MSC_CFG_offRDREG 10 -#define MSC_CFG_offADR24 11 -#define MSC_CFG_offINTLC 12 -#define MSC_CFG_offBASEV 13 -#define MSC_CFG_offNOD 16 -/* bit 13:31 reserved */ - -#define MSC_CFG_mskEDM ( 0x1 << MSC_CFG_offEDM ) -#define MSC_CFG_mskLMDMA ( 0x1 << MSC_CFG_offLMDMA ) -#define MSC_CFG_mskPFM ( 0x1 << MSC_CFG_offPFM ) -#define MSC_CFG_mskHSMP ( 0x1 << MSC_CFG_offHSMP ) -#define MSC_CFG_mskTRACE ( 0x1 << MSC_CFG_offTRACE ) -#define MSC_CFG_mskDIV ( 0x1 << MSC_CFG_offDIV ) -#define MSC_CFG_mskMAC ( 0x1 << MSC_CFG_offMAC ) -#define MSC_CFG_mskAUDIO ( 0x3 << MSC_CFG_offAUDIO ) -#define MSC_CFG_mskL2C ( 0x1 << MSC_CFG_offL2C ) -#define MSC_CFG_mskRDREG ( 0x1 << MSC_CFG_offRDREG ) -#define MSC_CFG_mskADR24 ( 0x1 << MSC_CFG_offADR24 ) -#define MSC_CFG_mskINTLC ( 0x1 << MSC_CFG_offINTLC ) -#define MSC_CFG_mskBASEV ( 0x7 << MSC_CFG_offBASEV ) -#define MSC_CFG_mskNOD ( 0x1 << MSC_CFG_offNOD ) - -/****************************************************************************** - * cr5: CORE_CFG (Core Identification Register) - *****************************************************************************/ -#define CORE_ID_offCOREID 0 -/* bit 4:31 reserved */ - -#define CORE_ID_mskCOREID ( 0xF << CORE_ID_offCOREID ) - -/****************************************************************************** - * cr6: FUCOP_EXIST (FPU and Coprocessor Existence Configuration Register) - *****************************************************************************/ -#define FUCOP_EXIST_offCP0EX 0 -#define FUCOP_EXIST_offCP1EX 1 -#define FUCOP_EXIST_offCP2EX 2 -#define FUCOP_EXIST_offCP3EX 3 -#define FUCOP_EXIST_offCP0ISFPU 31 - -#define FUCOP_EXIST_mskCP0EX ( 0x1 << FUCOP_EXIST_offCP0EX ) -#define FUCOP_EXIST_mskCP1EX ( 0x1 << FUCOP_EXIST_offCP1EX ) -#define FUCOP_EXIST_mskCP2EX ( 0x1 << FUCOP_EXIST_offCP2EX ) -#define FUCOP_EXIST_mskCP3EX ( 0x1 << FUCOP_EXIST_offCP3EX ) -#define FUCOP_EXIST_mskCP0ISFPU ( 0x1 << FUCOP_EXIST_offCP0ISFPU ) - -/****************************************************************************** - * ir0: PSW (Processor Status Word Register) - * ir1: IPSW (Interruption PSW Register) - * ir2: P_IPSW (Previous IPSW Register) - *****************************************************************************/ -#define PSW_offGIE 0 /* Global Interrupt Enable */ -#define PSW_offINTL 1 /* Interruption Stack Level */ -#define PSW_offPOM 3 /* Processor Operation Mode, User/Superuser */ -#define PSW_offBE 5 /* Endianness for data memory access, 1:MSB, 0:LSB */ -#define PSW_offIT 6 /* Enable instruction address translation */ -#define PSW_offDT 7 /* Enable data address translation */ -#define PSW_offIME 8 /* Instruction Machine Error flag */ -#define PSW_offDME 9 /* Data Machine Error flag */ -#define PSW_offDEX 10 /* Debug Exception */ -#define PSW_offHSS 11 /* Hardware Single Stepping */ -#define PSW_offDRBE 12 /* Device Register Endian Mode */ -#define PSW_offAEN 13 /* Audio ISA special feature */ -#define PSW_offWBNA 14 /* Write Back Non-Allocate */ -#define PSW_offIFCON 15 /* IFC On */ -#define PSW_offCPL 16 /* Current Priority Level */ -/* bit 19:31 reserved */ - -#define PSW_mskGIE ( 0x1 << PSW_offGIE ) -#define PSW_mskINTL ( 0x3 << PSW_offINTL ) -#define PSW_mskPOM ( 0x3 << PSW_offPOM ) -#define PSW_mskBE ( 0x1 << PSW_offBE ) -#define PSW_mskIT ( 0x1 << PSW_offIT ) -#define PSW_mskDT ( 0x1 << PSW_offDT ) -#define PSW_mskIME ( 0x1 << PSW_offIME ) -#define PSW_mskDME ( 0x1 << PSW_offDME ) -#define PSW_mskDEX ( 0x1 << PSW_offDEX ) -#define PSW_mskHSS ( 0x1 << PSW_offHSS ) -#define PSW_mskDRBE ( 0x1 << PSW_offDRBE ) -#define PSW_mskAEN ( 0x1 << PSW_offAEN ) -#define PSW_mskWBNA ( 0x1 << PSW_offWBNA ) -#define PSW_mskIFCON ( 0x1 << PSW_offIFCON ) -#define PSW_mskCPL ( 0x7 << PSW_offCPL ) - -#define PSW_SYSTEM ( 1 << PSW_offPOM ) -#define PSW_INTL_1 ( 1 << PSW_offINTL ) -#define PSW_CPL_NO ( 0 << PSW_offCPL ) -#define PSW_CPL_ANY ( 7 << PSW_offCPL ) - -#define PSW_clr (PSW_mskGIE|PSW_mskINTL|PSW_mskPOM|PSW_mskIT|PSW_mskDT|PSW_mskIME|PSW_mskWBNA) -#ifdef __NDS32_EB__ -#ifdef CONFIG_WBNA -#define PSW_init (PSW_mskWBNA|(1<<PSW_offINTL)|(1<<PSW_offPOM)|PSW_mskIT|PSW_mskDT|PSW_mskBE) -#else -#define PSW_init ((1<<PSW_offINTL)|(1<<PSW_offPOM)|PSW_mskIT|PSW_mskDT|PSW_mskBE) -#endif -#else -#ifdef CONFIG_WBNA -#define PSW_init (PSW_mskWBNA|(1<<PSW_offINTL)|(1<<PSW_offPOM)|PSW_mskIT|PSW_mskDT) -#else -#define PSW_init ((1<<PSW_offINTL)|(1<<PSW_offPOM)|PSW_mskIT|PSW_mskDT) -#endif -#endif -/****************************************************************************** - * ir3: IVB (Interruption Vector Base Register) - *****************************************************************************/ -/* bit 0:12 reserved */ -#define IVB_offNIVIC 1 /* Number of input for IVIC Controller */ -#define IVB_offIVIC_VER 11 /* IVIC Version */ -#define IVB_offEVIC 13 /* External Vector Interrupt Controller mode */ -#define IVB_offESZ 14 /* Size of each vector entry */ -#define IVB_offIVBASE 16 /* BasePA of interrupt vector table */ - -#define IVB_mskNIVIC ( 0x7 << IVB_offNIVIC ) -#define IVB_mskIVIC_VER ( 0x3 << IVB_offIVIC_VER ) -#define IVB_mskEVIC ( 0x1 << IVB_offEVIC ) -#define IVB_mskESZ ( 0x3 << IVB_offESZ ) -#define IVB_mskIVBASE ( 0xFFFF << IVB_offIVBASE ) - -#define IVB_valESZ4 0 -#define IVB_valESZ16 1 -#define IVB_valESZ64 2 -#define IVB_valESZ256 3 -/****************************************************************************** - * ir4: EVA (Exception Virtual Address Register) - * ir5: P_EVA (Previous EVA Register) - *****************************************************************************/ - - /* This register contains the VA that causes the exception */ - -/****************************************************************************** - * ir6: ITYPE (Interruption Type Register) - * ir7: P_ITYPE (Previous ITYPE Register) - *****************************************************************************/ -#define ITYPE_offETYPE 0 /* Exception Type */ -#define ITYPE_offINST 4 /* Exception caused by insn fetch or data access */ -/* bit 5:15 reserved */ -#define ITYPE_offVECTOR 5 /* Vector */ -#define ITYPE_offSWID 16 /* SWID of debugging exception */ -/* bit 31:31 reserved */ - -#define ITYPE_mskETYPE ( 0xF << ITYPE_offETYPE ) -#define ITYPE_mskINST ( 0x1 << ITYPE_offINST ) -#define ITYPE_mskVECTOR ( 0x7F << ITYPE_offVECTOR ) -#define ITYPE_mskSWID ( 0x7FFF << ITYPE_offSWID ) - -/* Additional definitions for ITYPE register */ -#define ITYPE_offSTYPE 16 /* Arithmetic Sub Type */ -#define ITYPE_offCPID 20 /* Co-Processor ID which generate the exception */ - -#define ITYPE_mskSTYPE ( 0xF << ITYPE_offSTYPE ) -#define ITYPE_mskCPID ( 0x3 << ITYPE_offCPID ) - -/* Additional definitions of ITYPE register for FPU */ -#define FPU_DISABLE_EXCEPTION (0x1 << ITYPE_offSTYPE) -#define FPU_EXCEPTION (0x2 << ITYPE_offSTYPE) -#define FPU_CPID 0 /* FPU Co-Processor ID is 0 */ - -#define NDS32_VECTOR_mskNONEXCEPTION 0x78 -#define NDS32_VECTOR_offEXCEPTION 8 -#define NDS32_VECTOR_offINTERRUPT 9 - -/* Interrupt vector entry */ -#define ENTRY_RESET_NMI 0 -#define ENTRY_TLB_FILL 1 -#define ENTRY_PTE_NOT_PRESENT 2 -#define ENTRY_TLB_MISC 3 -#define ENTRY_TLB_VLPT_MISS 4 -#define ENTRY_MACHINE_ERROR 5 -#define ENTRY_DEBUG_RELATED 6 -#define ENTRY_GENERAL_EXCPETION 7 -#define ENTRY_SYSCALL 8 - -/* PTE not present exception definition */ -#define ETYPE_NON_LEAF_PTE_NOT_PRESENT 0 -#define ETYPE_LEAF_PTE_NOT_PRESENT 1 - -/* General exception ETYPE definition */ -#define ETYPE_ALIGNMENT_CHECK 0 -#define ETYPE_RESERVED_INSTRUCTION 1 -#define ETYPE_TRAP 2 -#define ETYPE_ARITHMETIC 3 -#define ETYPE_PRECISE_BUS_ERROR 4 -#define ETYPE_IMPRECISE_BUS_ERROR 5 -#define ETYPE_COPROCESSOR 6 -#define ETYPE_RESERVED_VALUE 7 -#define ETYPE_NONEXISTENT_MEM_ADDRESS 8 -#define ETYPE_MPZIU_CONTROL 9 -#define ETYPE_NEXT_PRECISE_STACK_OFL 10 - -/* Kerenl reserves software ID */ -#define SWID_RAISE_INTERRUPT_LEVEL 0x1a /* SWID_RAISE_INTERRUPT_LEVEL is used to - * raise interrupt level for debug exception - */ - -/****************************************************************************** - * ir8: MERR (Machine Error Log Register) - *****************************************************************************/ -/* bit 0:30 reserved */ -#define MERR_offBUSERR 31 /* Bus error caused by a load insn */ - -#define MERR_mskBUSERR ( 0x1 << MERR_offBUSERR ) - -/****************************************************************************** - * ir9: IPC (Interruption Program Counter Register) - * ir10: P_IPC (Previous IPC Register) - * ir11: OIPC (Overflow Interruption Program Counter Register) - *****************************************************************************/ - - /* This is the shadow stack register of the Program Counter */ - -/****************************************************************************** - * ir12: P_P0 (Previous P0 Register) - * ir13: P_P1 (Previous P1 Register) - *****************************************************************************/ - - /* These are shadow registers of $p0 and $p1 */ - -/****************************************************************************** - * ir14: INT_MASK (Interruption Masking Register) - *****************************************************************************/ -#define INT_MASK_offH0IM 0 /* Hardware Interrupt 0 Mask bit */ -#define INT_MASK_offH1IM 1 /* Hardware Interrupt 1 Mask bit */ -#define INT_MASK_offH2IM 2 /* Hardware Interrupt 2 Mask bit */ -#define INT_MASK_offH3IM 3 /* Hardware Interrupt 3 Mask bit */ -#define INT_MASK_offH4IM 4 /* Hardware Interrupt 4 Mask bit */ -#define INT_MASK_offH5IM 5 /* Hardware Interrupt 5 Mask bit */ -/* bit 6:15 reserved */ -#define INT_MASK_offSIM 16 /* Software Interrupt Mask bit */ -/* bit 17:29 reserved */ -#define INT_MASK_offIDIVZE 30 /* Enable detection for Divide-By-Zero */ -#define INT_MASK_offDSSIM 31 /* Default Single Stepping Interruption Mask */ - -#define INT_MASK_mskH0IM ( 0x1 << INT_MASK_offH0IM ) -#define INT_MASK_mskH1IM ( 0x1 << INT_MASK_offH1IM ) -#define INT_MASK_mskH2IM ( 0x1 << INT_MASK_offH2IM ) -#define INT_MASK_mskH3IM ( 0x1 << INT_MASK_offH3IM ) -#define INT_MASK_mskH4IM ( 0x1 << INT_MASK_offH4IM ) -#define INT_MASK_mskH5IM ( 0x1 << INT_MASK_offH5IM ) -#define INT_MASK_mskSIM ( 0x1 << INT_MASK_offSIM ) -#define INT_MASK_mskIDIVZE ( 0x1 << INT_MASK_offIDIVZE ) -#define INT_MASK_mskDSSIM ( 0x1 << INT_MASK_offDSSIM ) - -#define INT_MASK_INITAIAL_VAL (INT_MASK_mskDSSIM|INT_MASK_mskIDIVZE) - -/****************************************************************************** - * ir15: INT_PEND (Interrupt Pending Register) - *****************************************************************************/ -#define INT_PEND_offH0I 0 /* Hardware Interrupt 0 pending bit */ -#define INT_PEND_offH1I 1 /* Hardware Interrupt 1 pending bit */ -#define INT_PEND_offH2I 2 /* Hardware Interrupt 2 pending bit */ -#define INT_PEND_offH3I 3 /* Hardware Interrupt 3 pending bit */ -#define INT_PEND_offH4I 4 /* Hardware Interrupt 4 pending bit */ -#define INT_PEND_offH5I 5 /* Hardware Interrupt 5 pending bit */ - -#define INT_PEND_offCIPL 0 /* Current Interrupt Priority Level */ - -/* bit 6:15 reserved */ -#define INT_PEND_offSWI 16 /* Software Interrupt pending bit */ -/* bit 17:31 reserved */ - -#define INT_PEND_mskH0I ( 0x1 << INT_PEND_offH0I ) -#define INT_PEND_mskH1I ( 0x1 << INT_PEND_offH1I ) -#define INT_PEND_mskH2I ( 0x1 << INT_PEND_offH2I ) -#define INT_PEND_mskH3I ( 0x1 << INT_PEND_offH3I ) -#define INT_PEND_mskH4I ( 0x1 << INT_PEND_offH4I ) -#define INT_PEND_mskH5I ( 0x1 << INT_PEND_offH5I ) -#define INT_PEND_mskCIPL ( 0x1 << INT_PEND_offCIPL ) -#define INT_PEND_mskSWI ( 0x1 << INT_PEND_offSWI ) - -/****************************************************************************** - * mr0: MMU_CTL (MMU Control Register) - *****************************************************************************/ -#define MMU_CTL_offD 0 /* Default minimum page size */ -#define MMU_CTL_offNTC0 1 /* Non-Translated Cachebility of partition 0 */ -#define MMU_CTL_offNTC1 3 /* Non-Translated Cachebility of partition 1 */ -#define MMU_CTL_offNTC2 5 /* Non-Translated Cachebility of partition 2 */ -#define MMU_CTL_offNTC3 7 /* Non-Translated Cachebility of partition 3 */ -#define MMU_CTL_offTBALCK 9 /* TLB all-lock resolution scheme */ -#define MMU_CTL_offMPZIU 10 /* Multiple Page Size In Use bit */ -#define MMU_CTL_offNTM0 11 /* Non-Translated VA to PA of partition 0 */ -#define MMU_CTL_offNTM1 13 /* Non-Translated VA to PA of partition 1 */ -#define MMU_CTL_offNTM2 15 /* Non-Translated VA to PA of partition 2 */ -#define MMU_CTL_offNTM3 17 /* Non-Translated VA to PA of partition 3 */ -#define MMU_CTL_offUNA 23 /* Unaligned access */ -/* bit 24:31 reserved */ - -#define MMU_CTL_mskD ( 0x1 << MMU_CTL_offD ) -#define MMU_CTL_mskNTC0 ( 0x3 << MMU_CTL_offNTC0 ) -#define MMU_CTL_mskNTC1 ( 0x3 << MMU_CTL_offNTC1 ) -#define MMU_CTL_mskNTC2 ( 0x3 << MMU_CTL_offNTC2 ) -#define MMU_CTL_mskNTC3 ( 0x3 << MMU_CTL_offNTC3 ) -#define MMU_CTL_mskTBALCK ( 0x1 << MMU_CTL_offTBALCK ) -#define MMU_CTL_mskMPZIU ( 0x1 << MMU_CTL_offMPZIU ) -#define MMU_CTL_mskNTM0 ( 0x3 << MMU_CTL_offNTM0 ) -#define MMU_CTL_mskNTM1 ( 0x3 << MMU_CTL_offNTM1 ) -#define MMU_CTL_mskNTM2 ( 0x3 << MMU_CTL_offNTM2 ) -#define MMU_CTL_mskNTM3 ( 0x3 << MMU_CTL_offNTM3 ) - -#define MMU_CTL_D4KB 0 -#define MMU_CTL_D8KB 1 -#define MMU_CTL_UNA ( 0x1 << MMU_CTL_offUNA ) - -#define MMU_CTL_CACHEABLE_NON 0 -#define MMU_CTL_CACHEABLE_WB 2 -#define MMU_CTL_CACHEABLE_WT 3 - -/****************************************************************************** - * mr1: L1_PPTB (L1 Physical Page Table Base Register) - *****************************************************************************/ -#define L1_PPTB_offNV 0 /* Enable Hardware Page Table Walker (HPTWK) */ -/* bit 1:11 reserved */ -#define L1_PPTB_offBASE 12 /* First level physical page table base address */ - -#define L1_PPTB_mskNV ( 0x1 << L1_PPTB_offNV ) -#define L1_PPTB_mskBASE ( 0xFFFFF << L1_PPTB_offBASE ) - -/****************************************************************************** - * mr2: TLB_VPN (TLB Access VPN Register) - *****************************************************************************/ -/* bit 0:11 reserved */ -#define TLB_VPN_offVPN 12 /* Virtual Page Number */ - -#define TLB_VPN_mskVPN ( 0xFFFFF << TLB_VPN_offVPN ) - -/****************************************************************************** - * mr3: TLB_DATA (TLB Access Data Register) - *****************************************************************************/ -#define TLB_DATA_offV 0 /* PTE is valid and present */ -#define TLB_DATA_offM 1 /* Page read/write access privilege */ -#define TLB_DATA_offD 4 /* Dirty bit */ -#define TLB_DATA_offX 5 /* Executable bit */ -#define TLB_DATA_offA 6 /* Access bit */ -#define TLB_DATA_offG 7 /* Global page (shared across contexts) */ -#define TLB_DATA_offC 8 /* Cacheability atribute */ -/* bit 11:11 reserved */ -#define TLB_DATA_offPPN 12 /* Phisical Page Number */ - -#define TLB_DATA_mskV ( 0x1 << TLB_DATA_offV ) -#define TLB_DATA_mskM ( 0x7 << TLB_DATA_offM ) -#define TLB_DATA_mskD ( 0x1 << TLB_DATA_offD ) -#define TLB_DATA_mskX ( 0x1 << TLB_DATA_offX ) -#define TLB_DATA_mskA ( 0x1 << TLB_DATA_offA ) -#define TLB_DATA_mskG ( 0x1 << TLB_DATA_offG ) -#define TLB_DATA_mskC ( 0x7 << TLB_DATA_offC ) -#define TLB_DATA_mskPPN ( 0xFFFFF << TLB_DATA_offPPN ) - -#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH -#define TLB_DATA_kernel_text_attr (TLB_DATA_mskV|TLB_DATA_mskM|TLB_DATA_mskD|TLB_DATA_mskX|TLB_DATA_mskG|TLB_DATA_mskC) -#else -#define TLB_DATA_kernel_text_attr (TLB_DATA_mskV|TLB_DATA_mskM|TLB_DATA_mskD|TLB_DATA_mskX|TLB_DATA_mskG|(0x6 << TLB_DATA_offC)) -#endif - -/****************************************************************************** - * mr4: TLB_MISC (TLB Access Misc Register) - *****************************************************************************/ -#define TLB_MISC_offACC_PSZ 0 /* Page size of a PTE entry */ -#define TLB_MISC_offCID 4 /* Context id */ -/* bit 13:31 reserved */ - -#define TLB_MISC_mskACC_PSZ ( 0xF << TLB_MISC_offACC_PSZ ) -#define TLB_MISC_mskCID ( 0x1FF << TLB_MISC_offCID ) - -/****************************************************************************** - * mr5: VLPT_IDX (Virtual Linear Page Table Index Register) - *****************************************************************************/ -#define VLPT_IDX_offZERO 0 /* Always 0 */ -#define VLPT_IDX_offEVPN 2 /* Exception Virtual Page Number */ -#define VLPT_IDX_offVLPTB 22 /* Base VA of VLPT */ - -#define VLPT_IDX_mskZERO ( 0x3 << VLPT_IDX_offZERO ) -#define VLPT_IDX_mskEVPN ( 0xFFFFF << VLPT_IDX_offEVPN ) -#define VLPT_IDX_mskVLPTB ( 0x3FF << VLPT_IDX_offVLPTB ) - -/****************************************************************************** - * mr6: ILMB (Instruction Local Memory Base Register) - *****************************************************************************/ -#define ILMB_offIEN 0 /* Enable ILM */ -#define ILMB_offILMSZ 1 /* Size of ILM */ -/* bit 5:19 reserved */ -#define ILMB_offIBPA 20 /* Base PA of ILM */ - -#define ILMB_mskIEN ( 0x1 << ILMB_offIEN ) -#define ILMB_mskILMSZ ( 0xF << ILMB_offILMSZ ) -#define ILMB_mskIBPA ( 0xFFF << ILMB_offIBPA ) - -/****************************************************************************** - * mr7: DLMB (Data Local Memory Base Register) - *****************************************************************************/ -#define DLMB_offDEN 0 /* Enable DLM */ -#define DLMB_offDLMSZ 1 /* Size of DLM */ -#define DLMB_offDBM 5 /* Enable Double-Buffer Mode for DLM */ -#define DLMB_offDBB 6 /* Double-buffer bank which can be accessed by the processor */ -/* bit 7:19 reserved */ -#define DLMB_offDBPA 20 /* Base PA of DLM */ - -#define DLMB_mskDEN ( 0x1 << DLMB_offDEN ) -#define DLMB_mskDLMSZ ( 0xF << DLMB_offDLMSZ ) -#define DLMB_mskDBM ( 0x1 << DLMB_offDBM ) -#define DLMB_mskDBB ( 0x1 << DLMB_offDBB ) -#define DLMB_mskDBPA ( 0xFFF << DLMB_offDBPA ) - -/****************************************************************************** - * mr8: CACHE_CTL (Cache Control Register) - *****************************************************************************/ -#define CACHE_CTL_offIC_EN 0 /* Enable I-cache */ -#define CACHE_CTL_offDC_EN 1 /* Enable D-cache */ -#define CACHE_CTL_offICALCK 2 /* I-cache all-lock resolution scheme */ -#define CACHE_CTL_offDCALCK 3 /* D-cache all-lock resolution scheme */ -#define CACHE_CTL_offDCCWF 4 /* Enable D-cache Critical Word Forwarding */ -#define CACHE_CTL_offDCPMW 5 /* Enable D-cache concurrent miss and write-back processing */ -/* bit 6:31 reserved */ - -#define CACHE_CTL_mskIC_EN ( 0x1 << CACHE_CTL_offIC_EN ) -#define CACHE_CTL_mskDC_EN ( 0x1 << CACHE_CTL_offDC_EN ) -#define CACHE_CTL_mskICALCK ( 0x1 << CACHE_CTL_offICALCK ) -#define CACHE_CTL_mskDCALCK ( 0x1 << CACHE_CTL_offDCALCK ) -#define CACHE_CTL_mskDCCWF ( 0x1 << CACHE_CTL_offDCCWF ) -#define CACHE_CTL_mskDCPMW ( 0x1 << CACHE_CTL_offDCPMW ) - -/****************************************************************************** - * mr9: HSMP_SADDR (High Speed Memory Port Starting Address) - *****************************************************************************/ -#define HSMP_SADDR_offEN 0 /* Enable control bit for the High Speed Memory port */ -/* bit 1:19 reserved */ - -#define HSMP_SADDR_offRANGE 1 /* Denote the address range (only defined in HSMP v2 ) */ -#define HSMP_SADDR_offSADDR 20 /* Starting base PA of the High Speed Memory Port region */ - -#define HSMP_SADDR_mskEN ( 0x1 << HSMP_SADDR_offEN ) -#define HSMP_SADDR_mskRANGE ( 0xFFF << HSMP_SADDR_offRANGE ) -#define HSMP_SADDR_mskSADDR ( 0xFFF << HSMP_SADDR_offSADDR ) - -/****************************************************************************** - * mr10: HSMP_EADDR (High Speed Memory Port Ending Address) - *****************************************************************************/ -/* bit 0:19 reserved */ -#define HSMP_EADDR_offEADDR 20 - -#define HSMP_EADDR_mskEADDR ( 0xFFF << HSMP_EADDR_offEADDR ) - -/****************************************************************************** - * dr0+(n*5): BPCn (n=0-7) (Breakpoint Control Register) - *****************************************************************************/ -#define BPC_offWP 0 /* Configuration of BPAn */ -#define BPC_offEL 1 /* Enable BPAn */ -#define BPC_offS 2 /* Data address comparison for a store instruction */ -#define BPC_offP 3 /* Compared data address is PA */ -#define BPC_offC 4 /* CID value is compared with the BPCIDn register */ -#define BPC_offBE0 5 /* Enable byte mask for the comparison with register */ -#define BPC_offBE1 6 /* Enable byte mask for the comparison with register */ -#define BPC_offBE2 7 /* Enable byte mask for the comparison with register */ -#define BPC_offBE3 8 /* Enable byte mask for the comparison with register */ -#define BPC_offT 9 /* Enable breakpoint Embedded Tracer triggering operation */ - -#define BPC_mskWP ( 0x1 << BPC_offWP ) -#define BPC_mskEL ( 0x1 << BPC_offEL ) -#define BPC_mskS ( 0x1 << BPC_offS ) -#define BPC_mskP ( 0x1 << BPC_offP ) -#define BPC_mskC ( 0x1 << BPC_offC ) -#define BPC_mskBE0 ( 0x1 << BPC_offBE0 ) -#define BPC_mskBE1 ( 0x1 << BPC_offBE1 ) -#define BPC_mskBE2 ( 0x1 << BPC_offBE2 ) -#define BPC_mskBE3 ( 0x1 << BPC_offBE3 ) -#define BPC_mskT ( 0x1 << BPC_offT ) - -/****************************************************************************** - * dr1+(n*5): BPAn (n=0-7) (Breakpoint Address Register) - *****************************************************************************/ - - /* These registers contain break point address */ - -/****************************************************************************** - * dr2+(n*5): BPAMn (n=0-7) (Breakpoint Address Mask Register) - *****************************************************************************/ - - /* These registerd contain the address comparison mask for the BPAn register */ - -/****************************************************************************** - * dr3+(n*5): BPVn (n=0-7) Breakpoint Data Value Register - *****************************************************************************/ - - /* The BPVn register contains the data value that will be compared with the - * incoming load/store data value */ - -/****************************************************************************** - * dr4+(n*5): BPCIDn (n=0-7) (Breakpoint Context ID Register) - *****************************************************************************/ -#define BPCID_offCID 0 /* CID that will be compared with a process's CID */ -/* bit 9:31 reserved */ - -#define BPCID_mskCID ( 0x1FF << BPCID_offCID ) - -/****************************************************************************** - * dr40: EDM_CFG (EDM Configuration Register) - *****************************************************************************/ -#define EDM_CFG_offBC 0 /* Number of hardware breakpoint sets implemented */ -#define EDM_CFG_offDIMU 3 /* Debug Instruction Memory Unit exists */ -/* bit 4:15 reserved */ -#define EDM_CFG_offVER 16 /* EDM version */ - -#define EDM_CFG_mskBC ( 0x7 << EDM_CFG_offBC ) -#define EDM_CFG_mskDIMU ( 0x1 << EDM_CFG_offDIMU ) -#define EDM_CFG_mskVER ( 0xFFFF << EDM_CFG_offVER ) - -/****************************************************************************** - * dr41: EDMSW (EDM Status Word) - *****************************************************************************/ -#define EDMSW_offWV 0 /* Write Valid */ -#define EDMSW_offRV 1 /* Read Valid */ -#define EDMSW_offDE 2 /* Debug exception has occurred for this core */ -/* bit 3:31 reserved */ - -#define EDMSW_mskWV ( 0x1 << EDMSW_offWV ) -#define EDMSW_mskRV ( 0x1 << EDMSW_offRV ) -#define EDMSW_mskDE ( 0x1 << EDMSW_offDE ) - -/****************************************************************************** - * dr42: EDM_CTL (EDM Control Register) - *****************************************************************************/ -/* bit 0:30 reserved */ -#define EDM_CTL_offV3_EDM_MODE 6 /* EDM compatibility control bit */ -#define EDM_CTL_offDEH_SEL 31 /* Controls where debug exception is directed to */ - -#define EDM_CTL_mskV3_EDM_MODE ( 0x1 << EDM_CTL_offV3_EDM_MODE ) -#define EDM_CTL_mskDEH_SEL ( 0x1 << EDM_CTL_offDEH_SEL ) - -/****************************************************************************** - * dr43: EDM_DTR (EDM Data Transfer Register) - *****************************************************************************/ - - /* This is used to exchange data between the embedded EDM logic - * and the processor core */ - -/****************************************************************************** - * dr44: BPMTC (Breakpoint Match Trigger Counter Register) - *****************************************************************************/ -#define BPMTC_offBPMTC 0 /* Breakpoint match trigger counter value */ -/* bit 16:31 reserved */ - -#define BPMTC_mskBPMTC ( 0xFFFF << BPMTC_offBPMTC ) - -/****************************************************************************** - * dr45: DIMBR (Debug Instruction Memory Base Register) - *****************************************************************************/ -/* bit 0:11 reserved */ -#define DIMBR_offDIMB 12 /* Base address of the Debug Instruction Memory (DIM) */ -#define DIMBR_mskDIMB ( 0xFFFFF << DIMBR_offDIMB ) - -/****************************************************************************** - * dr46: TECR0(Trigger Event Control register 0) - * dr47: TECR1 (Trigger Event Control register 1) - *****************************************************************************/ -#define TECR_offBP 0 /* Controld which BP is used as a trigger source */ -#define TECR_offNMI 8 /* Use NMI as a trigger source */ -#define TECR_offHWINT 9 /* Corresponding interrupt is used as a trigger source */ -#define TECR_offEVIC 15 /* Enable HWINT as a trigger source in EVIC mode */ -#define TECR_offSYS 16 /* Enable SYSCALL instruction as a trigger source */ -#define TECR_offDBG 17 /* Enable debug exception as a trigger source */ -#define TECR_offMRE 18 /* Enable MMU related exception as a trigger source */ -#define TECR_offE 19 /* An exception is used as a trigger source */ -/* bit 20:30 reserved */ -#define TECR_offL 31 /* Link/Cascade TECR0 trigger event to TECR1 trigger event */ - -#define TECR_mskBP ( 0xFF << TECR_offBP ) -#define TECR_mskNMI ( 0x1 << TECR_offBNMI ) -#define TECR_mskHWINT ( 0x3F << TECR_offBHWINT ) -#define TECR_mskEVIC ( 0x1 << TECR_offBEVIC ) -#define TECR_mskSYS ( 0x1 << TECR_offBSYS ) -#define TECR_mskDBG ( 0x1 << TECR_offBDBG ) -#define TECR_mskMRE ( 0x1 << TECR_offBMRE ) -#define TECR_mskE ( 0x1 << TECR_offE ) -#define TECR_mskL ( 0x1 << TECR_offL ) - -/****************************************************************************** - * pfr0-2: PFMC0-2 (Performance Counter Register 0-2) - *****************************************************************************/ - - /* These registers contains performance event count */ - -/****************************************************************************** - * pfr3: PFM_CTL (Performance Counter Control Register) - *****************************************************************************/ -#define PFM_CTL_offEN0 0 /* Enable PFMC0 */ -#define PFM_CTL_offEN1 1 /* Enable PFMC1 */ -#define PFM_CTL_offEN2 2 /* Enable PFMC2 */ -#define PFM_CTL_offIE0 3 /* Enable interrupt for PFMC0 */ -#define PFM_CTL_offIE1 4 /* Enable interrupt for PFMC1 */ -#define PFM_CTL_offIE2 5 /* Enable interrupt for PFMC2 */ -#define PFM_CTL_offOVF0 6 /* Overflow bit of PFMC0 */ -#define PFM_CTL_offOVF1 7 /* Overflow bit of PFMC1 */ -#define PFM_CTL_offOVF2 8 /* Overflow bit of PFMC2 */ -#define PFM_CTL_offKS0 9 /* Enable superuser mode event counting for PFMC0 */ -#define PFM_CTL_offKS1 10 /* Enable superuser mode event counting for PFMC1 */ -#define PFM_CTL_offKS2 11 /* Enable superuser mode event counting for PFMC2 */ -#define PFM_CTL_offKU0 12 /* Enable user mode event counting for PFMC0 */ -#define PFM_CTL_offKU1 13 /* Enable user mode event counting for PFMC1 */ -#define PFM_CTL_offKU2 14 /* Enable user mode event counting for PFMC2 */ -#define PFM_CTL_offSEL0 15 /* The event selection for PFMC0 */ -#define PFM_CTL_offSEL1 16 /* The event selection for PFMC1 */ -#define PFM_CTL_offSEL2 22 /* The event selection for PFMC2 */ -/* bit 28:31 reserved */ - -#define PFM_CTL_mskEN0 ( 0x01 << PFM_CTL_offEN0 ) -#define PFM_CTL_mskEN1 ( 0x01 << PFM_CTL_offEN1 ) -#define PFM_CTL_mskEN2 ( 0x01 << PFM_CTL_offEN2 ) -#define PFM_CTL_mskIE0 ( 0x01 << PFM_CTL_offIE0 ) -#define PFM_CTL_mskIE1 ( 0x01 << PFM_CTL_offIE1 ) -#define PFM_CTL_mskIE2 ( 0x01 << PFM_CTL_offIE2 ) -#define PFM_CTL_mskOVF0 ( 0x01 << PFM_CTL_offOVF0 ) -#define PFM_CTL_mskOVF1 ( 0x01 << PFM_CTL_offOVF1 ) -#define PFM_CTL_mskOVF2 ( 0x01 << PFM_CTL_offOVF2 ) -#define PFM_CTL_mskKS0 ( 0x01 << PFM_CTL_offKS0 ) -#define PFM_CTL_mskKS1 ( 0x01 << PFM_CTL_offKS1 ) -#define PFM_CTL_mskKS2 ( 0x01 << PFM_CTL_offKS2 ) -#define PFM_CTL_mskKU0 ( 0x01 << PFM_CTL_offKU0 ) -#define PFM_CTL_mskKU1 ( 0x01 << PFM_CTL_offKU1 ) -#define PFM_CTL_mskKU2 ( 0x01 << PFM_CTL_offKU2 ) -#define PFM_CTL_mskSEL0 ( 0x01 << PFM_CTL_offSEL0 ) -#define PFM_CTL_mskSEL1 ( 0x3F << PFM_CTL_offSEL1 ) -#define PFM_CTL_mskSEL2 ( 0x3F << PFM_CTL_offSEL2 ) - -/****************************************************************************** - * SDZ_CTL (Structure Downsizing Control Register) - *****************************************************************************/ -#define SDZ_CTL_offICDZ 0 /* I-cache downsizing control */ -#define SDZ_CTL_offDCDZ 3 /* D-cache downsizing control */ -#define SDZ_CTL_offMTBDZ 6 /* MTLB downsizing control */ -#define SDZ_CTL_offBTBDZ 9 /* Branch Target Table downsizing control */ -/* bit 12:31 reserved */ -#define SDZ_CTL_mskICDZ ( 0x07 << SDZ_CTL_offICDZ ) -#define SDZ_CTL_mskDCDZ ( 0x07 << SDZ_CTL_offDCDZ ) -#define SDZ_CTL_mskMTBDZ ( 0x07 << SDZ_CTL_offMTBDZ ) -#define SDZ_CTL_mskBTBDZ ( 0x07 << SDZ_CTL_offBTBDZ ) - -/****************************************************************************** - * N13MISC_CTL (N13 Miscellaneous Control Register) - *****************************************************************************/ -#define N13MISC_CTL_offBTB 0 /* Disable Branch Target Buffer */ -#define N13MISC_CTL_offRTP 1 /* Disable Return Target Predictor */ -#define N13MISC_CTL_offPTEPF 2 /* Disable HPTWK L2 PTE pefetch */ -#define N13MISC_CTL_offSP_SHADOW_EN 4 /* Enable shadow stack pointers */ -#define MISC_CTL_offHWPRE 11 /* Enable HardWare PREFETCH */ -/* bit 6, 9:31 reserved */ - -#define N13MISC_CTL_makBTB ( 0x1 << N13MISC_CTL_offBTB ) -#define N13MISC_CTL_makRTP ( 0x1 << N13MISC_CTL_offRTP ) -#define N13MISC_CTL_makPTEPF ( 0x1 << N13MISC_CTL_offPTEPF ) -#define N13MISC_CTL_makSP_SHADOW_EN ( 0x1 << N13MISC_CTL_offSP_SHADOW_EN ) -#define MISC_CTL_makHWPRE_EN ( 0x1 << MISC_CTL_offHWPRE ) - -#ifdef CONFIG_HW_PRE -#define MISC_init (N13MISC_CTL_makBTB|N13MISC_CTL_makRTP|N13MISC_CTL_makSP_SHADOW_EN|MISC_CTL_makHWPRE_EN) -#else -#define MISC_init (N13MISC_CTL_makBTB|N13MISC_CTL_makRTP|N13MISC_CTL_makSP_SHADOW_EN) -#endif - -/****************************************************************************** - * PRUSR_ACC_CTL (Privileged Resource User Access Control Registers) - *****************************************************************************/ -#define PRUSR_ACC_CTL_offDMA_EN 0 /* Allow user mode access of DMA registers */ -#define PRUSR_ACC_CTL_offPFM_EN 1 /* Allow user mode access of PFM registers */ - -#define PRUSR_ACC_CTL_mskDMA_EN ( 0x1 << PRUSR_ACC_CTL_offDMA_EN ) -#define PRUSR_ACC_CTL_mskPFM_EN ( 0x1 << PRUSR_ACC_CTL_offPFM_EN ) - -/****************************************************************************** - * dmar0: DMA_CFG (DMA Configuration Register) - *****************************************************************************/ -#define DMA_CFG_offNCHN 0 /* The number of DMA channels implemented */ -#define DMA_CFG_offUNEA 2 /* Un-aligned External Address transfer feature */ -#define DMA_CFG_off2DET 3 /* 2-D Element Transfer feature */ -/* bit 4:15 reserved */ -#define DMA_CFG_offVER 16 /* DMA architecture and implementation version */ - -#define DMA_CFG_mskNCHN ( 0x3 << DMA_CFG_offNCHN ) -#define DMA_CFG_mskUNEA ( 0x1 << DMA_CFG_offUNEA ) -#define DMA_CFG_msk2DET ( 0x1 << DMA_CFG_off2DET ) -#define DMA_CFG_mskVER ( 0xFFFF << DMA_CFG_offVER ) - -/****************************************************************************** - * dmar1: DMA_GCSW (DMA Global Control and Status Word Register) - *****************************************************************************/ -#define DMA_GCSW_offC0STAT 0 /* DMA channel 0 state */ -#define DMA_GCSW_offC1STAT 3 /* DMA channel 1 state */ -/* bit 6:11 reserved */ -#define DMA_GCSW_offC0INT 12 /* DMA channel 0 generate interrupt */ -#define DMA_GCSW_offC1INT 13 /* DMA channel 1 generate interrupt */ -/* bit 14:30 reserved */ -#define DMA_GCSW_offEN 31 /* Enable DMA engine */ - -#define DMA_GCSW_mskC0STAT ( 0x7 << DMA_GCSW_offC0STAT ) -#define DMA_GCSW_mskC1STAT ( 0x7 << DMA_GCSW_offC1STAT ) -#define DMA_GCSW_mskC0INT ( 0x1 << DMA_GCSW_offC0INT ) -#define DMA_GCSW_mskC1INT ( 0x1 << DMA_GCSW_offC1INT ) -#define DMA_GCSW_mskEN ( 0x1 << DMA_GCSW_offEN ) - -/****************************************************************************** - * dmar2: DMA_CHNSEL (DMA Channel Selection Register) - *****************************************************************************/ -#define DMA_CHNSEL_offCHAN 0 /* Selected channel number */ -/* bit 2:31 reserved */ - -#define DMA_CHNSEL_mskCHAN ( 0x3 << DMA_CHNSEL_offCHAN ) - -/****************************************************************************** - * dmar3: DMA_ACT (DMA Action Register) - *****************************************************************************/ -#define DMA_ACT_offACMD 0 /* DMA Action Command */ -/* bit 2:31 reserved */ -#define DMA_ACT_mskACMD ( 0x3 << DMA_ACT_offACMD ) - -/****************************************************************************** - * dmar4: DMA_SETUP (DMA Setup Register) - *****************************************************************************/ -#define DMA_SETUP_offLM 0 /* Local Memory Selection */ -#define DMA_SETUP_offTDIR 1 /* Transfer Direction */ -#define DMA_SETUP_offTES 2 /* Transfer Element Size */ -#define DMA_SETUP_offESTR 4 /* External memory transfer Stride */ -#define DMA_SETUP_offCIE 16 /* Interrupt Enable on Completion */ -#define DMA_SETUP_offSIE 17 /* Interrupt Enable on explicit Stop */ -#define DMA_SETUP_offEIE 18 /* Interrupt Enable on Error */ -#define DMA_SETUP_offUE 19 /* Enable the Un-aligned External Address */ -#define DMA_SETUP_off2DE 20 /* Enable the 2-D External Transfer */ -#define DMA_SETUP_offCOA 21 /* Transfer Coalescable */ -/* bit 22:31 reserved */ - -#define DMA_SETUP_mskLM ( 0x1 << DMA_SETUP_offLM ) -#define DMA_SETUP_mskTDIR ( 0x1 << DMA_SETUP_offTDIR ) -#define DMA_SETUP_mskTES ( 0x3 << DMA_SETUP_offTES ) -#define DMA_SETUP_mskESTR ( 0xFFF << DMA_SETUP_offESTR ) -#define DMA_SETUP_mskCIE ( 0x1 << DMA_SETUP_offCIE ) -#define DMA_SETUP_mskSIE ( 0x1 << DMA_SETUP_offSIE ) -#define DMA_SETUP_mskEIE ( 0x1 << DMA_SETUP_offEIE ) -#define DMA_SETUP_mskUE ( 0x1 << DMA_SETUP_offUE ) -#define DMA_SETUP_msk2DE ( 0x1 << DMA_SETUP_off2DE ) -#define DMA_SETUP_mskCOA ( 0x1 << DMA_SETUP_offCOA ) - -/****************************************************************************** - * dmar5: DMA_ISADDR (DMA Internal Start Address Register) - *****************************************************************************/ -#define DMA_ISADDR_offISADDR 0 /* Internal Start Address */ -/* bit 20:31 reserved */ -#define DMA_ISADDR_mskISADDR ( 0xFFFFF << DMA_ISADDR_offISADDR ) - -/****************************************************************************** - * dmar6: DMA_ESADDR (DMA External Start Address Register) - *****************************************************************************/ -/* This register holds External Start Address */ - -/****************************************************************************** - * dmar7: DMA_TCNT (DMA Transfer Element Count Register) - *****************************************************************************/ -#define DMA_TCNT_offTCNT 0 /* DMA transfer element count */ -/* bit 18:31 reserved */ -#define DMA_TCNT_mskTCNT ( 0x3FFFF << DMA_TCNT_offTCNT ) - -/****************************************************************************** - * dmar8: DMA_STATUS (DMA Status Register) - *****************************************************************************/ -#define DMA_STATUS_offSTAT 0 /* DMA channel state */ -#define DMA_STATUS_offSTUNA 3 /* Un-aligned error on External Stride value */ -#define DMA_STATUS_offDERR 4 /* DMA Transfer Disruption Error */ -#define DMA_STATUS_offEUNA 5 /* Un-aligned error on the External address */ -#define DMA_STATUS_offIUNA 6 /* Un-aligned error on the Internal address */ -#define DMA_STATUS_offIOOR 7 /* Out-Of-Range error on the Internal address */ -#define DMA_STATUS_offEBUS 8 /* Bus Error on an External DMA transfer */ -#define DMA_STATUS_offESUP 9 /* DMA setup error */ -/* bit 10:31 reserved */ - -#define DMA_STATUS_mskSTAT ( 0x7 << DMA_STATUS_offSTAT ) -#define DMA_STATUS_mskSTUNA ( 0x1 << DMDMA_STATUS_offSTUNA ) -#define DMA_STATUS_mskDERR ( 0x1 << DMDMA_STATUS_offDERR ) -#define DMA_STATUS_mskEUNA ( 0x1 << DMDMA_STATUS_offEUNA ) -#define DMA_STATUS_mskIUNA ( 0x1 << DMDMA_STATUS_offIUNA ) -#define DMA_STATUS_mskIOOR ( 0x1 << DMDMA_STATUS_offIOOR ) -#define DMA_STATUS_mskEBUS ( 0x1 << DMDMA_STATUS_offEBUS ) -#define DMA_STATUS_mskESUP ( 0x1 << DMDMA_STATUS_offESUP ) - -/****************************************************************************** - * dmar9: DMA_2DSET (DMA 2D Setup Register) - *****************************************************************************/ -#define DMA_2DSET_offWECNT 0 /* The Width Element Count for a 2-D region */ -#define DMA_2DSET_offHTSTR 16 /* The Height Stride for a 2-D region */ - -#define DMA_2DSET_mskHTSTR ( 0xFFFF << DMA_2DSET_offHTSTR ) -#define DMA_2DSET_mskWECNT ( 0xFFFF << DMA_2DSET_offWECNT ) - -/****************************************************************************** - * dmar10: DMA_2DSCTL (DMA 2D Startup Control Register) - *****************************************************************************/ -#define DMA_2DSCTL_offSTWECNT 0 /* Startup Width Element Count for a 2-D region */ -/* bit 16:31 reserved */ - -#define DMA_2DSCTL_mskSTWECNT ( 0xFFFF << DMA_2DSCTL_offSTWECNT ) - -/****************************************************************************** - * fpcsr: FPCSR (Floating-Point Control Status Register) - *****************************************************************************/ -#define FPCSR_offRM 0 -#define FPCSR_offIVO 2 -#define FPCSR_offDBZ 3 -#define FPCSR_offOVF 4 -#define FPCSR_offUDF 5 -#define FPCSR_offIEX 6 -#define FPCSR_offIVOE 7 -#define FPCSR_offDBZE 8 -#define FPCSR_offOVFE 9 -#define FPCSR_offUDFE 10 -#define FPCSR_offIEXE 11 -#define FPCSR_offDNZ 12 -#define FPCSR_offIVOT 13 -#define FPCSR_offDBZT 14 -#define FPCSR_offOVFT 15 -#define FPCSR_offUDFT 16 -#define FPCSR_offIEXT 17 -#define FPCSR_offDNIT 18 -#define FPCSR_offRIT 19 - -#define FPCSR_mskRM ( 0x3 << FPCSR_offRM ) -#define FPCSR_mskIVO ( 0x1 << FPCSR_offIVO ) -#define FPCSR_mskDBZ ( 0x1 << FPCSR_offDBZ ) -#define FPCSR_mskOVF ( 0x1 << FPCSR_offOVF ) -#define FPCSR_mskUDF ( 0x1 << FPCSR_offUDF ) -#define FPCSR_mskIEX ( 0x1 << FPCSR_offIEX ) -#define FPCSR_mskIVOE ( 0x1 << FPCSR_offIVOE ) -#define FPCSR_mskDBZE ( 0x1 << FPCSR_offDBZE ) -#define FPCSR_mskOVFE ( 0x1 << FPCSR_offOVFE ) -#define FPCSR_mskUDFE ( 0x1 << FPCSR_offUDFE ) -#define FPCSR_mskIEXE ( 0x1 << FPCSR_offIEXE ) -#define FPCSR_mskDNZ ( 0x1 << FPCSR_offDNZ ) -#define FPCSR_mskIVOT ( 0x1 << FPCSR_offIVOT ) -#define FPCSR_mskDBZT ( 0x1 << FPCSR_offDBZT ) -#define FPCSR_mskOVFT ( 0x1 << FPCSR_offOVFT ) -#define FPCSR_mskUDFT ( 0x1 << FPCSR_offUDFT ) -#define FPCSR_mskIEXT ( 0x1 << FPCSR_offIEXT ) -#define FPCSR_mskDNIT ( 0x1 << FPCSR_offDNIT ) -#define FPCSR_mskRIT ( 0x1 << FPCSR_offRIT ) -#define FPCSR_mskALL (FPCSR_mskIVO | FPCSR_mskDBZ | FPCSR_mskOVF | FPCSR_mskUDF | FPCSR_mskIEX) -#define FPCSR_mskALLE_NO_UDF_IEXE (FPCSR_mskIVOE | FPCSR_mskDBZE | FPCSR_mskOVFE) -#define FPCSR_mskALLE (FPCSR_mskIVOE | FPCSR_mskDBZE | FPCSR_mskOVFE | FPCSR_mskUDFE | FPCSR_mskIEXE) -#define FPCSR_mskALLT (FPCSR_mskIVOT | FPCSR_mskDBZT | FPCSR_mskOVFT | FPCSR_mskUDFT | FPCSR_mskIEXT |FPCSR_mskDNIT | FPCSR_mskRIT) - -/****************************************************************************** - * fpcfg: FPCFG (Floating-Point Configuration Register) - *****************************************************************************/ -#define FPCFG_offSP 0 -#define FPCFG_offDP 1 -#define FPCFG_offFREG 2 -#define FPCFG_offFMA 4 -#define FPCFG_offIMVER 22 -#define FPCFG_offAVER 27 - -#define FPCFG_mskSP ( 0x1 << FPCFG_offSP ) -#define FPCFG_mskDP ( 0x1 << FPCFG_offDP ) -#define FPCFG_mskFREG ( 0x3 << FPCFG_offFREG ) -#define FPCFG_mskFMA ( 0x1 << FPCFG_offFMA ) -#define FPCFG_mskIMVER ( 0x1F << FPCFG_offIMVER ) -#define FPCFG_mskAVER ( 0x1F << FPCFG_offAVER ) - -/* 8 Single precision or 4 double precision registers are available */ -#define SP8_DP4_reg 0 -/* 16 Single precision or 8 double precision registers are available */ -#define SP16_DP8_reg 1 -/* 32 Single precision or 16 double precision registers are available */ -#define SP32_DP16_reg 2 -/* 32 Single precision or 32 double precision registers are available */ -#define SP32_DP32_reg 3 - -/****************************************************************************** - * fucpr: FUCOP_CTL (FPU and Coprocessor Enable Control Register) - *****************************************************************************/ -#define FUCOP_CTL_offCP0EN 0 -#define FUCOP_CTL_offCP1EN 1 -#define FUCOP_CTL_offCP2EN 2 -#define FUCOP_CTL_offCP3EN 3 -#define FUCOP_CTL_offAUEN 31 - -#define FUCOP_CTL_mskCP0EN ( 0x1 << FUCOP_CTL_offCP0EN ) -#define FUCOP_CTL_mskCP1EN ( 0x1 << FUCOP_CTL_offCP1EN ) -#define FUCOP_CTL_mskCP2EN ( 0x1 << FUCOP_CTL_offCP2EN ) -#define FUCOP_CTL_mskCP3EN ( 0x1 << FUCOP_CTL_offCP3EN ) -#define FUCOP_CTL_mskAUEN ( 0x1 << FUCOP_CTL_offAUEN ) - -#endif /* __NDS32_BITFIELD_H__ */ diff --git a/arch/nds32/include/asm/cache.h b/arch/nds32/include/asm/cache.h deleted file mode 100644 index fc3c41b59169..000000000000 --- a/arch/nds32/include/asm/cache.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_CACHE_H__ -#define __NDS32_CACHE_H__ - -#define L1_CACHE_BYTES 32 -#define L1_CACHE_SHIFT 5 - -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES - -#endif /* __NDS32_CACHE_H__ */ diff --git a/arch/nds32/include/asm/cache_info.h b/arch/nds32/include/asm/cache_info.h deleted file mode 100644 index e89d8078f3a6..000000000000 --- a/arch/nds32/include/asm/cache_info.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -struct cache_info { - unsigned char ways; - unsigned char line_size; - unsigned short sets; - unsigned short size; -#if defined(CONFIG_CPU_CACHE_ALIASING) - unsigned short aliasing_num; - unsigned int aliasing_mask; -#endif -}; diff --git a/arch/nds32/include/asm/cacheflush.h b/arch/nds32/include/asm/cacheflush.h deleted file mode 100644 index c2a222ebfa2a..000000000000 --- a/arch/nds32/include/asm/cacheflush.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_CACHEFLUSH_H__ -#define __NDS32_CACHEFLUSH_H__ - -#include <linux/mm.h> - -#define PG_dcache_dirty PG_arch_1 - -void flush_icache_range(unsigned long start, unsigned long end); -#define flush_icache_range flush_icache_range - -void flush_icache_page(struct vm_area_struct *vma, struct page *page); -#define flush_icache_page flush_icache_page - -#ifdef CONFIG_CPU_CACHE_ALIASING -void flush_cache_mm(struct mm_struct *mm); -void flush_cache_dup_mm(struct mm_struct *mm); -void flush_cache_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end); -void flush_cache_page(struct vm_area_struct *vma, - unsigned long addr, unsigned long pfn); -void flush_cache_kmaps(void); -void flush_cache_vmap(unsigned long start, unsigned long end); -void flush_cache_vunmap(unsigned long start, unsigned long end); - -#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -void flush_dcache_page(struct page *page); -void copy_to_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, void *dst, void *src, int len); -void copy_from_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, void *dst, void *src, int len); - -#define ARCH_HAS_FLUSH_ANON_PAGE -void flush_anon_page(struct vm_area_struct *vma, - struct page *page, unsigned long vaddr); - -#define ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 1 -void flush_kernel_vmap_range(void *addr, int size); -void invalidate_kernel_vmap_range(void *addr, int size); -#define flush_dcache_mmap_lock(mapping) xa_lock_irq(&(mapping)->i_pages) -#define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&(mapping)->i_pages) - -#else -void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long addr, int len); -#define flush_icache_user_page flush_icache_user_page - -#include <asm-generic/cacheflush.h> -#endif - -#endif /* __NDS32_CACHEFLUSH_H__ */ diff --git a/arch/nds32/include/asm/current.h b/arch/nds32/include/asm/current.h deleted file mode 100644 index 65d30096142b..000000000000 --- a/arch/nds32/include/asm/current.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASM_NDS32_CURRENT_H -#define _ASM_NDS32_CURRENT_H - -#ifndef __ASSEMBLY__ -register struct task_struct *current asm("$r25"); -#endif /* __ASSEMBLY__ */ -#define tsk $r25 - -#endif /* _ASM_NDS32_CURRENT_H */ diff --git a/arch/nds32/include/asm/delay.h b/arch/nds32/include/asm/delay.h deleted file mode 100644 index 56ea3894f8f8..000000000000 --- a/arch/nds32/include/asm/delay.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_DELAY_H__ -#define __NDS32_DELAY_H__ - -#include <asm/param.h> - -/* There is no clocksource cycle counter in the CPU. */ -static inline void __delay(unsigned long loops) -{ - __asm__ __volatile__(".align 2\n" - "1:\n" - "\taddi\t%0, %0, -1\n" - "\tbgtz\t%0, 1b\n" - :"=r"(loops) - :"0"(loops)); -} - -static inline void __udelay(unsigned long usecs, unsigned long lpj) -{ - usecs *= (unsigned long)(((0x8000000000000000ULL / (500000 / HZ)) + - 0x80000000ULL) >> 32); - usecs = (unsigned long)(((unsigned long long)usecs * lpj) >> 32); - __delay(usecs); -} - -#define udelay(usecs) __udelay((usecs), loops_per_jiffy) - -/* make sure "usecs *= ..." in udelay do not overflow. */ -#if HZ >= 1000 -#define MAX_UDELAY_MS 1 -#elif HZ <= 200 -#define MAX_UDELAY_MS 5 -#else -#define MAX_UDELAY_MS (1000 / HZ) -#endif - -#endif diff --git a/arch/nds32/include/asm/elf.h b/arch/nds32/include/asm/elf.h deleted file mode 100644 index 1853dc89b8ac..000000000000 --- a/arch/nds32/include/asm/elf.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASMNDS32_ELF_H -#define __ASMNDS32_ELF_H - -/* - * ELF register definitions.. - */ - -#include <asm/ptrace.h> -#include <asm/fpu.h> -#include <linux/elf-em.h> - -typedef unsigned long elf_greg_t; -typedef unsigned long elf_freg_t[3]; - -extern unsigned int elf_hwcap; - -#define R_NDS32_NONE 0 -#define R_NDS32_16_RELA 19 -#define R_NDS32_32_RELA 20 -#define R_NDS32_9_PCREL_RELA 22 -#define R_NDS32_15_PCREL_RELA 23 -#define R_NDS32_17_PCREL_RELA 24 -#define R_NDS32_25_PCREL_RELA 25 -#define R_NDS32_HI20_RELA 26 -#define R_NDS32_LO12S3_RELA 27 -#define R_NDS32_LO12S2_RELA 28 -#define R_NDS32_LO12S1_RELA 29 -#define R_NDS32_LO12S0_RELA 30 -#define R_NDS32_SDA15S3_RELA 31 -#define R_NDS32_SDA15S2_RELA 32 -#define R_NDS32_SDA15S1_RELA 33 -#define R_NDS32_SDA15S0_RELA 34 -#define R_NDS32_GOT20 37 -#define R_NDS32_25_PLTREL 38 -#define R_NDS32_COPY 39 -#define R_NDS32_GLOB_DAT 40 -#define R_NDS32_JMP_SLOT 41 -#define R_NDS32_RELATIVE 42 -#define R_NDS32_GOTOFF 43 -#define R_NDS32_GOTPC20 44 -#define R_NDS32_GOT_HI20 45 -#define R_NDS32_GOT_LO12 46 -#define R_NDS32_GOTPC_HI20 47 -#define R_NDS32_GOTPC_LO12 48 -#define R_NDS32_GOTOFF_HI20 49 -#define R_NDS32_GOTOFF_LO12 50 -#define R_NDS32_INSN16 51 -#define R_NDS32_LABEL 52 -#define R_NDS32_LONGCALL1 53 -#define R_NDS32_LONGCALL2 54 -#define R_NDS32_LONGCALL3 55 -#define R_NDS32_LONGJUMP1 56 -#define R_NDS32_LONGJUMP2 57 -#define R_NDS32_LONGJUMP3 58 -#define R_NDS32_LOADSTORE 59 -#define R_NDS32_9_FIXED_RELA 60 -#define R_NDS32_15_FIXED_RELA 61 -#define R_NDS32_17_FIXED_RELA 62 -#define R_NDS32_25_FIXED_RELA 63 -#define R_NDS32_PLTREL_HI20 64 -#define R_NDS32_PLTREL_LO12 65 -#define R_NDS32_PLT_GOTREL_HI20 66 -#define R_NDS32_PLT_GOTREL_LO12 67 -#define R_NDS32_LO12S0_ORI_RELA 72 -#define R_NDS32_DWARF2_OP1_RELA 77 -#define R_NDS32_DWARF2_OP2_RELA 78 -#define R_NDS32_DWARF2_LEB_RELA 79 -#define R_NDS32_WORD_9_PCREL_RELA 94 -#define R_NDS32_LONGCALL4 107 -#define R_NDS32_RELA_NOP_MIX 192 -#define R_NDS32_RELA_NOP_MAX 255 - -#define ELF_NGREG (sizeof (struct user_pt_regs) / sizeof(elf_greg_t)) -#define ELF_CORE_COPY_REGS(dest, regs) \ - *(struct user_pt_regs *)&(dest) = (regs)->user_regs; - -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; - -/* Core file format: The core file is written in such a way that gdb - can understand it and provide useful information to the user (under - linux we use the 'trad-core' bfd). There are quite a number of - obstacles to being able to view the contents of the floating point - registers, and until these are solved you will not be able to view the - contents of them. Actually, you can read in the core file and look at - the contents of the user struct to find out what the floating point - registers contain. - The actual file contents are as follows: - UPAGE: 1 page consisting of a user struct that tells gdb what is present - in the file. Directly after this is a copy of the task_struct, which - is currently not used by gdb, but it may come in useful at some point. - All of the registers are stored as part of the upage. The upage should - always be only one page. - DATA: The data area is stored. We use current->end_text to - current->brk to pick up all of the user variables, plus any memory - that may have been malloced. No attempt is made to determine if a page - is demand-zero or if a page is totally unused, we just cover the entire - range. All of the addresses are rounded in such a way that an integral - number of pages is written. - STACK: We need the stack information in order to get a meaningful - backtrace. We need to write the data from (esp) to - current->start_stack, so we round each of these off in order to be able - to write an integer number of pages. - The minimum core file size is 3 pages, or 12288 bytes. -*/ - -struct user_fp { - unsigned long long fd_regs[32]; - unsigned long fpcsr; -}; - -typedef struct user_fp elf_fpregset_t; - -struct elf32_hdr; -#define elf_check_arch(x) ((x)->e_machine == EM_NDS32) - -/* - * These are used to set parameters in the core dumps. - */ -#define ELF_CLASS ELFCLASS32 -#ifdef __NDS32_EB__ -#define ELF_DATA ELFDATA2MSB -#else -#define ELF_DATA ELFDATA2LSB -#endif -#define ELF_ARCH EM_NDS32 -#define ELF_EXEC_PAGESIZE PAGE_SIZE - -/* This is the location that an ET_DYN program is loaded if exec'ed. Typical - use of this is to invoke "./ld.so someprog" to test out a new version of - the loader. We need to make sure that it is out of the way of the program - that it will "exec", and that there is sufficient room for the brk. */ - -#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) - -/* When the program starts, a1 contains a pointer to a function to be - registered with atexit, as per the SVR4 ABI. A value of 0 means we - have no such handler. */ -#define ELF_PLAT_INIT(_r, load_addr) (_r)->uregs[0] = 0 - -/* This yields a mask that user programs can use to figure out what - instruction set this cpu supports. */ - -#define ELF_HWCAP (elf_hwcap) - -#ifdef __KERNEL__ - -#define ELF_PLATFORM (NULL) - -/* Old NetWinder binaries were compiled in such a way that the iBCS - heuristic always trips on them. Until these binaries become uncommon - enough not to care, don't trust the `ibcs' flag here. In any case - there is no other ELF system currently supported by iBCS. - @@ Could print a warning message to encourage users to upgrade. */ -#define SET_PERSONALITY(ex) set_personality(PER_LINUX) - -#endif - - -#if IS_ENABLED(CONFIG_FPU) -#define FPU_AUX_ENT NEW_AUX_ENT(AT_FPUCW, FPCSR_INIT) -#else -#define FPU_AUX_ENT NEW_AUX_ENT(AT_IGNORE, 0) -#endif - -#define ARCH_DLINFO \ -do { \ - /* Optional FPU initialization */ \ - FPU_AUX_ENT; \ - \ - NEW_AUX_ENT(AT_SYSINFO_EHDR, \ - (elf_addr_t)current->mm->context.vdso); \ -} while (0) -#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 -struct linux_binprm; -int arch_setup_additional_pages(struct linux_binprm *, int); - -#endif diff --git a/arch/nds32/include/asm/fixmap.h b/arch/nds32/include/asm/fixmap.h deleted file mode 100644 index 2fa09a2de428..000000000000 --- a/arch/nds32/include/asm/fixmap.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_FIXMAP_H -#define __ASM_NDS32_FIXMAP_H - -#ifdef CONFIG_HIGHMEM -#include <linux/threads.h> -#include <asm/kmap_size.h> -#endif - -enum fixed_addresses { - FIX_HOLE, - FIX_KMAP_RESERVED, - FIX_KMAP_BEGIN, -#ifdef CONFIG_HIGHMEM - FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_MAX_IDX * NR_CPUS) - 1, -#endif - FIX_EARLYCON_MEM_BASE, - __end_of_fixed_addresses -}; -#define FIXADDR_TOP ((unsigned long) (-(16 * PAGE_SIZE))) -#define FIXADDR_SIZE ((__end_of_fixed_addresses) << PAGE_SHIFT) -#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) -#define FIXMAP_PAGE_IO __pgprot(PAGE_DEVICE) -void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); - -#include <asm-generic/fixmap.h> -#endif /* __ASM_NDS32_FIXMAP_H */ diff --git a/arch/nds32/include/asm/fpu.h b/arch/nds32/include/asm/fpu.h deleted file mode 100644 index 8294ed4aaa2c..000000000000 --- a/arch/nds32/include/asm/fpu.h +++ /dev/null @@ -1,126 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2005-2018 Andes Technology Corporation */ - -#ifndef __ASM_NDS32_FPU_H -#define __ASM_NDS32_FPU_H - -#if IS_ENABLED(CONFIG_FPU) -#ifndef __ASSEMBLY__ -#include <linux/sched/task_stack.h> -#include <linux/preempt.h> -#include <asm/ptrace.h> - -extern bool has_fpu; - -extern void save_fpu(struct task_struct *__tsk); -extern void load_fpu(const struct fpu_struct *fpregs); -extern bool do_fpu_exception(unsigned int subtype, struct pt_regs *regs); -extern int do_fpuemu(struct pt_regs *regs, struct fpu_struct *fpu); - -#define test_tsk_fpu(regs) (regs->fucop_ctl & FUCOP_CTL_mskCP0EN) - -/* - * Initially load the FPU with signalling NANS. This bit pattern - * has the property that no matter whether considered as single or as - * double precision, it still represents a signalling NAN. - */ - -#define sNAN64 0xFFFFFFFFFFFFFFFFULL -#define sNAN32 0xFFFFFFFFUL - -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) -/* - * Denormalized number is unsupported by nds32 FPU. Hence the operation - * is treated as underflow cases when the final result is a denormalized - * number. To enhance precision, underflow exception trap should be - * enabled by default and kerenl will re-execute it by fpu emulator - * when getting underflow exception. - */ -#define FPCSR_INIT (FPCSR_mskUDFE | FPCSR_mskIEXE) -#else -#define FPCSR_INIT 0x0UL -#endif - -extern const struct fpu_struct init_fpuregs; - -static inline void disable_ptreg_fpu(struct pt_regs *regs) -{ - regs->fucop_ctl &= ~FUCOP_CTL_mskCP0EN; -} - -static inline void enable_ptreg_fpu(struct pt_regs *regs) -{ - regs->fucop_ctl |= FUCOP_CTL_mskCP0EN; -} - -static inline void enable_fpu(void) -{ - unsigned long fucop_ctl; - - fucop_ctl = __nds32__mfsr(NDS32_SR_FUCOP_CTL) | FUCOP_CTL_mskCP0EN; - __nds32__mtsr(fucop_ctl, NDS32_SR_FUCOP_CTL); - __nds32__isb(); -} - -static inline void disable_fpu(void) -{ - unsigned long fucop_ctl; - - fucop_ctl = __nds32__mfsr(NDS32_SR_FUCOP_CTL) & ~FUCOP_CTL_mskCP0EN; - __nds32__mtsr(fucop_ctl, NDS32_SR_FUCOP_CTL); - __nds32__isb(); -} - -static inline void lose_fpu(void) -{ - preempt_disable(); -#if IS_ENABLED(CONFIG_LAZY_FPU) - if (last_task_used_math == current) { - last_task_used_math = NULL; -#else - if (test_tsk_fpu(task_pt_regs(current))) { -#endif - save_fpu(current); - } - disable_ptreg_fpu(task_pt_regs(current)); - preempt_enable(); -} - -static inline void own_fpu(void) -{ - preempt_disable(); -#if IS_ENABLED(CONFIG_LAZY_FPU) - if (last_task_used_math != current) { - if (last_task_used_math != NULL) - save_fpu(last_task_used_math); - load_fpu(¤t->thread.fpu); - last_task_used_math = current; - } -#else - if (!test_tsk_fpu(task_pt_regs(current))) { - load_fpu(¤t->thread.fpu); - } -#endif - enable_ptreg_fpu(task_pt_regs(current)); - preempt_enable(); -} - -#if !IS_ENABLED(CONFIG_LAZY_FPU) -static inline void unlazy_fpu(struct task_struct *tsk) -{ - preempt_disable(); - if (test_tsk_fpu(task_pt_regs(tsk))) - save_fpu(tsk); - preempt_enable(); -} -#endif /* !CONFIG_LAZY_FPU */ -static inline void clear_fpu(struct pt_regs *regs) -{ - preempt_disable(); - if (test_tsk_fpu(regs)) - disable_ptreg_fpu(regs); - preempt_enable(); -} -#endif /* CONFIG_FPU */ -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_NDS32_FPU_H */ diff --git a/arch/nds32/include/asm/fpuemu.h b/arch/nds32/include/asm/fpuemu.h deleted file mode 100644 index 63e7ef5f7969..000000000000 --- a/arch/nds32/include/asm/fpuemu.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2005-2018 Andes Technology Corporation */ - -#ifndef __ARCH_NDS32_FPUEMU_H -#define __ARCH_NDS32_FPUEMU_H - -/* - * single precision - */ - -void fadds(void *ft, void *fa, void *fb); -void fsubs(void *ft, void *fa, void *fb); -void fmuls(void *ft, void *fa, void *fb); -void fdivs(void *ft, void *fa, void *fb); -void fs2d(void *ft, void *fa); -void fs2si(void *ft, void *fa); -void fs2si_z(void *ft, void *fa); -void fs2ui(void *ft, void *fa); -void fs2ui_z(void *ft, void *fa); -void fsi2s(void *ft, void *fa); -void fui2s(void *ft, void *fa); -void fsqrts(void *ft, void *fa); -void fnegs(void *ft, void *fa); -int fcmps(void *ft, void *fa, void *fb, int cop); - -/* - * double precision - */ -void faddd(void *ft, void *fa, void *fb); -void fsubd(void *ft, void *fa, void *fb); -void fmuld(void *ft, void *fa, void *fb); -void fdivd(void *ft, void *fa, void *fb); -void fsqrtd(void *ft, void *fa); -void fd2s(void *ft, void *fa); -void fd2si(void *ft, void *fa); -void fd2si_z(void *ft, void *fa); -void fd2ui(void *ft, void *fa); -void fd2ui_z(void *ft, void *fa); -void fsi2d(void *ft, void *fa); -void fui2d(void *ft, void *fa); -void fnegd(void *ft, void *fa); -int fcmpd(void *ft, void *fa, void *fb, int cop); - -#endif /* __ARCH_NDS32_FPUEMU_H */ diff --git a/arch/nds32/include/asm/ftrace.h b/arch/nds32/include/asm/ftrace.h deleted file mode 100644 index 2f96cc96aa35..000000000000 --- a/arch/nds32/include/asm/ftrace.h +++ /dev/null @@ -1,46 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef __ASM_NDS32_FTRACE_H -#define __ASM_NDS32_FTRACE_H - -#ifdef CONFIG_FUNCTION_TRACER - -#define HAVE_FUNCTION_GRAPH_FP_TEST - -#define MCOUNT_ADDR ((unsigned long)(_mcount)) -/* mcount call is composed of three instructions: - * sethi + ori + jral - */ -#define MCOUNT_INSN_SIZE 12 - -extern void _mcount(unsigned long parent_ip); - -#ifdef CONFIG_DYNAMIC_FTRACE - -#define FTRACE_ADDR ((unsigned long)_ftrace_caller) - -#ifdef __NDS32_EL__ -#define INSN_NOP 0x09000040 -#define INSN_SIZE(insn) (((insn & 0x00000080) == 0) ? 4 : 2) -#define IS_SETHI(insn) ((insn & 0x000000fe) == 0x00000046) -#define ENDIAN_CONVERT(insn) be32_to_cpu(insn) -#else /* __NDS32_EB__ */ -#define INSN_NOP 0x40000009 -#define INSN_SIZE(insn) (((insn & 0x80000000) == 0) ? 4 : 2) -#define IS_SETHI(insn) ((insn & 0xfe000000) == 0x46000000) -#define ENDIAN_CONVERT(insn) (insn) -#endif - -extern void _ftrace_caller(unsigned long parent_ip); -static inline unsigned long ftrace_call_adjust(unsigned long addr) -{ - return addr; -} -struct dyn_arch_ftrace { -}; - -#endif /* CONFIG_DYNAMIC_FTRACE */ - -#endif /* CONFIG_FUNCTION_TRACER */ - -#endif /* __ASM_NDS32_FTRACE_H */ diff --git a/arch/nds32/include/asm/futex.h b/arch/nds32/include/asm/futex.h deleted file mode 100644 index 4223f473bd36..000000000000 --- a/arch/nds32/include/asm/futex.h +++ /dev/null @@ -1,101 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_FUTEX_H__ -#define __NDS32_FUTEX_H__ - -#include <linux/futex.h> -#include <linux/uaccess.h> -#include <asm/errno.h> - -#define __futex_atomic_ex_table(err_reg) \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 4f\n" \ - " .long 2b, 4f\n" \ - " .popsection\n" \ - " .pushsection .fixup,\"ax\"\n" \ - "4: move %0, " err_reg "\n" \ - " b 3b\n" \ - " .popsection" - -#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ - smp_mb(); \ - asm volatile( \ - " movi $ta, #0\n" \ - "1: llw %1, [%2+$ta]\n" \ - " " insn "\n" \ - "2: scw %0, [%2+$ta]\n" \ - " beqz %0, 1b\n" \ - " movi %0, #0\n" \ - "3:\n" \ - __futex_atomic_ex_table("%4") \ - : "=&r" (ret), "=&r" (oldval) \ - : "r" (uaddr), "r" (oparg), "i" (-EFAULT) \ - : "cc", "memory") -static inline int -futex_atomic_cmpxchg_inatomic(u32 * uval, u32 __user * uaddr, - u32 oldval, u32 newval) -{ - int ret = 0; - u32 val, tmp, flags; - - if (!access_ok(uaddr, sizeof(u32))) - return -EFAULT; - - smp_mb(); - asm volatile (" movi $ta, #0\n" - "1: llw %1, [%6 + $ta]\n" - " sub %3, %1, %4\n" - " cmovz %2, %5, %3\n" - " cmovn %2, %1, %3\n" - "2: scw %2, [%6 + $ta]\n" - " beqz %2, 1b\n" - "3:\n " __futex_atomic_ex_table("%7") - :"+&r"(ret), "=&r"(val), "=&r"(tmp), "=&r"(flags) - :"r"(oldval), "r"(newval), "r"(uaddr), "i"(-EFAULT) - :"$ta", "memory"); - smp_mb(); - - *uval = val; - return ret; -} - -static inline int -arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) -{ - int oldval = 0, ret; - - if (!access_ok(uaddr, sizeof(u32))) - return -EFAULT; - switch (op) { - case FUTEX_OP_SET: - __futex_atomic_op("move %0, %3", ret, oldval, tmp, uaddr, - oparg); - break; - case FUTEX_OP_ADD: - __futex_atomic_op("add %0, %1, %3", ret, oldval, tmp, uaddr, - oparg); - break; - case FUTEX_OP_OR: - __futex_atomic_op("or %0, %1, %3", ret, oldval, tmp, uaddr, - oparg); - break; - case FUTEX_OP_ANDN: - __futex_atomic_op("and %0, %1, %3", ret, oldval, tmp, uaddr, - ~oparg); - break; - case FUTEX_OP_XOR: - __futex_atomic_op("xor %0, %1, %3", ret, oldval, tmp, uaddr, - oparg); - break; - default: - ret = -ENOSYS; - } - - if (!ret) - *oval = oldval; - - return ret; -} -#endif /* __NDS32_FUTEX_H__ */ diff --git a/arch/nds32/include/asm/highmem.h b/arch/nds32/include/asm/highmem.h deleted file mode 100644 index 16159a8716f2..000000000000 --- a/arch/nds32/include/asm/highmem.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASM_HIGHMEM_H -#define _ASM_HIGHMEM_H - -#include <asm/proc-fns.h> -#include <asm/fixmap.h> - -/* - * Right now we initialize only a single pte table. It can be extended - * easily, subsequent pte tables have to be allocated in one physical - * chunk of RAM. - */ -/* - * Ordering is (from lower to higher memory addresses): - * - * high_memory - * Persistent kmap area - * PKMAP_BASE - * fixed_addresses - * FIXADDR_START - * FIXADDR_TOP - * Vmalloc area - * VMALLOC_START - * VMALLOC_END - */ -#define PKMAP_BASE ((FIXADDR_START - PGDIR_SIZE) & (PGDIR_MASK)) -#define LAST_PKMAP PTRS_PER_PTE -#define LAST_PKMAP_MASK (LAST_PKMAP - 1) -#define PKMAP_NR(virt) (((virt) - (PKMAP_BASE)) >> PAGE_SHIFT) -#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT)) - -static inline void flush_cache_kmaps(void) -{ - cpu_dcache_wbinval_all(); -} - -/* declarations for highmem.c */ -extern unsigned long highstart_pfn, highend_pfn; - -extern pte_t *pkmap_page_table; - -extern void kmap_init(void); - -/* - * FIXME: The below looks broken vs. a kmap_atomic() in task context which - * is interupted and another kmap_atomic() happens in interrupt context. - * But what do I know about nds32. -- tglx - */ -#define arch_kmap_local_post_map(vaddr, pteval) \ - do { \ - __nds32__tlbop_inv(vaddr); \ - __nds32__mtsr_dsb(vaddr, NDS32_SR_TLB_VPN); \ - __nds32__tlbop_rwr(pteval); \ - __nds32__isb(); \ - } while (0) - -#define arch_kmap_local_pre_unmap(vaddr) \ - do { \ - __nds32__tlbop_inv(vaddr); \ - __nds32__isb(); \ - } while (0) - -#endif diff --git a/arch/nds32/include/asm/io.h b/arch/nds32/include/asm/io.h deleted file mode 100644 index e57378d04006..000000000000 --- a/arch/nds32/include/asm/io.h +++ /dev/null @@ -1,84 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_IO_H -#define __ASM_NDS32_IO_H - -#include <linux/types.h> - -#define __raw_writeb __raw_writeb -static inline void __raw_writeb(u8 val, volatile void __iomem *addr) -{ - asm volatile("sbi %0, [%1]" : : "r" (val), "r" (addr)); -} - -#define __raw_writew __raw_writew -static inline void __raw_writew(u16 val, volatile void __iomem *addr) -{ - asm volatile("shi %0, [%1]" : : "r" (val), "r" (addr)); -} - -#define __raw_writel __raw_writel -static inline void __raw_writel(u32 val, volatile void __iomem *addr) -{ - asm volatile("swi %0, [%1]" : : "r" (val), "r" (addr)); -} - -#define __raw_readb __raw_readb -static inline u8 __raw_readb(const volatile void __iomem *addr) -{ - u8 val; - - asm volatile("lbi %0, [%1]" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readw __raw_readw -static inline u16 __raw_readw(const volatile void __iomem *addr) -{ - u16 val; - - asm volatile("lhi %0, [%1]" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readl __raw_readl -static inline u32 __raw_readl(const volatile void __iomem *addr) -{ - u32 val; - - asm volatile("lwi %0, [%1]" : "=r" (val) : "r" (addr)); - return val; -} - -#define __iormb() rmb() -#define __iowmb() wmb() - -/* - * {read,write}{b,w,l,q}_relaxed() are like the regular version, but - * are not guaranteed to provide ordering against spinlocks or memory - * accesses. - */ - -#define readb_relaxed(c) ({ u8 __v = __raw_readb(c); __v; }) -#define readw_relaxed(c) ({ u16 __v = le16_to_cpu((__force __le16)__raw_readw(c)); __v; }) -#define readl_relaxed(c) ({ u32 __v = le32_to_cpu((__force __le32)__raw_readl(c)); __v; }) -#define writeb_relaxed(v,c) ((void)__raw_writeb((v),(c))) -#define writew_relaxed(v,c) ((void)__raw_writew((__force u16)cpu_to_le16(v),(c))) -#define writel_relaxed(v,c) ((void)__raw_writel((__force u32)cpu_to_le32(v),(c))) - -/* - * {read,write}{b,w,l,q}() access little endian memory and return result in - * native endianness. - */ -#define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(); __v; }) -#define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(); __v; }) -#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(); __v; }) - -#define writeb(v,c) ({ __iowmb(); writeb_relaxed((v),(c)); }) -#define writew(v,c) ({ __iowmb(); writew_relaxed((v),(c)); }) -#define writel(v,c) ({ __iowmb(); writel_relaxed((v),(c)); }) - -#include <asm-generic/io.h> - -#endif /* __ASM_NDS32_IO_H */ diff --git a/arch/nds32/include/asm/irqflags.h b/arch/nds32/include/asm/irqflags.h deleted file mode 100644 index 51ef800bb301..000000000000 --- a/arch/nds32/include/asm/irqflags.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <asm/nds32.h> -#include <nds32_intrinsic.h> - -#define arch_local_irq_disable() \ - GIE_DISABLE(); - -#define arch_local_irq_enable() \ - GIE_ENABLE(); -static inline unsigned long arch_local_irq_save(void) -{ - unsigned long flags; - flags = __nds32__mfsr(NDS32_SR_PSW) & PSW_mskGIE; - GIE_DISABLE(); - return flags; -} - -static inline unsigned long arch_local_save_flags(void) -{ - unsigned long flags; - flags = __nds32__mfsr(NDS32_SR_PSW) & PSW_mskGIE; - return flags; -} - -static inline void arch_local_irq_restore(unsigned long flags) -{ - if(flags) - GIE_ENABLE(); -} - -static inline int arch_irqs_disabled_flags(unsigned long flags) -{ - return !flags; -} - -static inline int arch_irqs_disabled(void) -{ - return arch_irqs_disabled_flags(arch_local_save_flags()); -} diff --git a/arch/nds32/include/asm/l2_cache.h b/arch/nds32/include/asm/l2_cache.h deleted file mode 100644 index 3ea48e19e6de..000000000000 --- a/arch/nds32/include/asm/l2_cache.h +++ /dev/null @@ -1,137 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef L2_CACHE_H -#define L2_CACHE_H - -/* CCTL_CMD_OP */ -#define L2_CA_CONF_OFF 0x0 -#define L2_IF_CONF_OFF 0x4 -#define L2CC_SETUP_OFF 0x8 -#define L2CC_PROT_OFF 0xC -#define L2CC_CTRL_OFF 0x10 -#define L2_INT_EN_OFF 0x20 -#define L2_STA_OFF 0x24 -#define RDERR_ADDR_OFF 0x28 -#define WRERR_ADDR_OFF 0x2c -#define EVDPTERR_ADDR_OFF 0x30 -#define IMPL3ERR_ADDR_OFF 0x34 -#define L2_CNT0_CTRL_OFF 0x40 -#define L2_EVNT_CNT0_OFF 0x44 -#define L2_CNT1_CTRL_OFF 0x48 -#define L2_EVNT_CNT1_OFF 0x4c -#define L2_CCTL_CMD_OFF 0x60 -#define L2_CCTL_STATUS_OFF 0x64 -#define L2_LINE_TAG_OFF 0x68 -#define L2_LINE_DPT_OFF 0x70 - -#define CCTL_CMD_L2_IX_INVAL 0x0 -#define CCTL_CMD_L2_PA_INVAL 0x1 -#define CCTL_CMD_L2_IX_WB 0x2 -#define CCTL_CMD_L2_PA_WB 0x3 -#define CCTL_CMD_L2_PA_WBINVAL 0x5 -#define CCTL_CMD_L2_SYNC 0xa - -/* CCTL_CMD_TYPE */ -#define CCTL_SINGLE_CMD 0 -#define CCTL_BLOCK_CMD 0x10 -#define CCTL_ALL_CMD 0x10 - -/****************************************************************************** - * L2_CA_CONF (Cache architecture configuration) - *****************************************************************************/ -#define L2_CA_CONF_offL2SET 0 -#define L2_CA_CONF_offL2WAY 4 -#define L2_CA_CONF_offL2CLSZ 8 -#define L2_CA_CONF_offL2DW 11 -#define L2_CA_CONF_offL2PT 14 -#define L2_CA_CONF_offL2VER 16 - -#define L2_CA_CONF_mskL2SET (0xFUL << L2_CA_CONF_offL2SET) -#define L2_CA_CONF_mskL2WAY (0xFUL << L2_CA_CONF_offL2WAY) -#define L2_CA_CONF_mskL2CLSZ (0x7UL << L2_CA_CONF_offL2CLSZ) -#define L2_CA_CONF_mskL2DW (0x7UL << L2_CA_CONF_offL2DW) -#define L2_CA_CONF_mskL2PT (0x3UL << L2_CA_CONF_offL2PT) -#define L2_CA_CONF_mskL2VER (0xFFFFUL << L2_CA_CONF_offL2VER) - -/****************************************************************************** - * L2CC_SETUP (L2CC Setup register) - *****************************************************************************/ -#define L2CC_SETUP_offPART 0 -#define L2CC_SETUP_mskPART (0x3UL << L2CC_SETUP_offPART) -#define L2CC_SETUP_offDDLATC 4 -#define L2CC_SETUP_mskDDLATC (0x3UL << L2CC_SETUP_offDDLATC) -#define L2CC_SETUP_offTDLATC 8 -#define L2CC_SETUP_mskTDLATC (0x3UL << L2CC_SETUP_offTDLATC) - -/****************************************************************************** - * L2CC_PROT (L2CC Protect register) - *****************************************************************************/ -#define L2CC_PROT_offMRWEN 31 -#define L2CC_PROT_mskMRWEN (0x1UL << L2CC_PROT_offMRWEN) - -/****************************************************************************** - * L2_CCTL_STATUS_Mn (The L2CCTL command working status for Master n) - *****************************************************************************/ -#define L2CC_CTRL_offEN 31 -#define L2CC_CTRL_mskEN (0x1UL << L2CC_CTRL_offEN) - -/****************************************************************************** - * L2_CCTL_STATUS_Mn (The L2CCTL command working status for Master n) - *****************************************************************************/ -#define L2_CCTL_STATUS_offCMD_COMP 31 -#define L2_CCTL_STATUS_mskCMD_COMP (0x1 << L2_CCTL_STATUS_offCMD_COMP) - -extern void __iomem *atl2c_base; -#include <linux/smp.h> -#include <asm/io.h> -#include <asm/bitfield.h> - -#define L2C_R_REG(offset) readl(atl2c_base + offset) -#define L2C_W_REG(offset, value) writel(value, atl2c_base + offset) - -#define L2_CMD_RDY() \ - do{;}while((L2C_R_REG(L2_CCTL_STATUS_OFF) & L2_CCTL_STATUS_mskCMD_COMP) == 0) - -static inline unsigned long L2_CACHE_SET(void) -{ - return 64 << ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2SET) >> - L2_CA_CONF_offL2SET); -} - -static inline unsigned long L2_CACHE_WAY(void) -{ - return 1 + - ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2WAY) >> - L2_CA_CONF_offL2WAY); -} - -static inline unsigned long L2_CACHE_LINE_SIZE(void) -{ - - return 4 << ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2CLSZ) >> - L2_CA_CONF_offL2CLSZ); -} - -static inline unsigned long GET_L2CC_CTRL_CPU(unsigned long cpu) -{ - if (cpu == smp_processor_id()) - return L2C_R_REG(L2CC_CTRL_OFF); - return L2C_R_REG(L2CC_CTRL_OFF + (cpu << 8)); -} - -static inline void SET_L2CC_CTRL_CPU(unsigned long cpu, unsigned long val) -{ - if (cpu == smp_processor_id()) - L2C_W_REG(L2CC_CTRL_OFF, val); - else - L2C_W_REG(L2CC_CTRL_OFF + (cpu << 8), val); -} - -static inline unsigned long GET_L2CC_STATUS_CPU(unsigned long cpu) -{ - if (cpu == smp_processor_id()) - return L2C_R_REG(L2_CCTL_STATUS_OFF); - return L2C_R_REG(L2_CCTL_STATUS_OFF + (cpu << 8)); -} -#endif diff --git a/arch/nds32/include/asm/linkage.h b/arch/nds32/include/asm/linkage.h deleted file mode 100644 index a696469abb70..000000000000 --- a/arch/nds32/include/asm/linkage.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_LINKAGE_H -#define __ASM_LINKAGE_H - -/* This file is required by include/linux/linkage.h */ -#define __ALIGN .align 2 -#define __ALIGN_STR ".align 2" - -#endif diff --git a/arch/nds32/include/asm/memory.h b/arch/nds32/include/asm/memory.h deleted file mode 100644 index 62faafbc28e4..000000000000 --- a/arch/nds32/include/asm/memory.h +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_MEMORY_H -#define __ASM_NDS32_MEMORY_H - -#include <linux/compiler.h> -#include <linux/sizes.h> - -#ifndef __ASSEMBLY__ -#include <asm/page.h> -#endif - -#ifndef PHYS_OFFSET -#define PHYS_OFFSET (0x0) -#endif - -/* - * TASK_SIZE - the maximum size of a user space task. - * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area - */ -#define TASK_SIZE ((CONFIG_PAGE_OFFSET) - (SZ_32M)) -#define TASK_UNMAPPED_BASE ALIGN(TASK_SIZE / 3, SZ_32M) -#define PAGE_OFFSET (CONFIG_PAGE_OFFSET) - -/* - * Physical vs virtual RAM address space conversion. These are - * private definitions which should NOT be used outside memory.h - * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. - */ -#ifndef __virt_to_phys -#define __virt_to_phys(x) ((x) - PAGE_OFFSET + PHYS_OFFSET) -#define __phys_to_virt(x) ((x) - PHYS_OFFSET + PAGE_OFFSET) -#endif - -/* - * The module space lives between the addresses given by TASK_SIZE - * and PAGE_OFFSET - it must be within 32MB of the kernel text. - */ -#define MODULES_END (PAGE_OFFSET) -#define MODULES_VADDR (MODULES_END - SZ_32M) - -#if TASK_SIZE > MODULES_VADDR -#error Top of user space clashes with start of module space -#endif - -#ifndef __ASSEMBLY__ - -/* - * PFNs are used to describe any physical page; this means - * PFN 0 == physical address 0. - * - * This is the PFN of the first RAM page in the kernel - * direct-mapped view. We assume this is the first page - * of RAM in the mem_map as well. - */ -#define PHYS_PFN_OFFSET (PHYS_OFFSET >> PAGE_SHIFT) - -/* - * Drivers should NOT use these either. - */ -#define __pa(x) __virt_to_phys((unsigned long)(x)) -#define __va(x) ((void *)__phys_to_virt((unsigned long)(x))) - -/* - * Conversion between a struct page and a physical address. - * - * Note: when converting an unknown physical address to a - * struct page, the resulting pointer must be validated - * using VALID_PAGE(). It must return an invalid struct page - * for any physical address not corresponding to a system - * RAM address. - * - * pfn_valid(pfn) indicates whether a PFN number is valid - * - * virt_to_page(k) convert a _valid_ virtual address to struct page * - * virt_addr_valid(k) indicates whether a virtual address is valid - */ -#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET -#define pfn_valid(pfn) ((pfn) >= PHYS_PFN_OFFSET && (pfn) < (PHYS_PFN_OFFSET + max_mapnr)) - -#define virt_to_page(kaddr) (pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)) -#define virt_addr_valid(kaddr) ((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) - -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) - -#endif - -#include <asm-generic/memory_model.h> - -#endif diff --git a/arch/nds32/include/asm/mmu.h b/arch/nds32/include/asm/mmu.h deleted file mode 100644 index 89d63afee455..000000000000 --- a/arch/nds32/include/asm/mmu.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_MMU_H -#define __NDS32_MMU_H - -typedef struct { - unsigned int id; - void *vdso; -} mm_context_t; - -#endif diff --git a/arch/nds32/include/asm/mmu_context.h b/arch/nds32/include/asm/mmu_context.h deleted file mode 100644 index c651bc8cacdc..000000000000 --- a/arch/nds32/include/asm/mmu_context.h +++ /dev/null @@ -1,62 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_MMU_CONTEXT_H -#define __ASM_NDS32_MMU_CONTEXT_H - -#include <linux/spinlock.h> -#include <asm/tlbflush.h> -#include <asm/proc-fns.h> -#include <asm-generic/mm_hooks.h> - -#define init_new_context init_new_context -static inline int -init_new_context(struct task_struct *tsk, struct mm_struct *mm) -{ - mm->context.id = 0; - return 0; -} - -#define CID_BITS 9 -extern spinlock_t cid_lock; -extern unsigned int cpu_last_cid; - -static inline void __new_context(struct mm_struct *mm) -{ - unsigned int cid; - unsigned long flags; - - spin_lock_irqsave(&cid_lock, flags); - cid = cpu_last_cid; - cpu_last_cid += 1 << TLB_MISC_offCID; - if (cpu_last_cid == 0) - cpu_last_cid = 1 << TLB_MISC_offCID << CID_BITS; - - if ((cid & TLB_MISC_mskCID) == 0) - flush_tlb_all(); - spin_unlock_irqrestore(&cid_lock, flags); - - mm->context.id = cid; -} - -static inline void check_context(struct mm_struct *mm) -{ - if (unlikely - ((mm->context.id ^ cpu_last_cid) >> TLB_MISC_offCID >> CID_BITS)) - __new_context(mm); -} - -static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) -{ - unsigned int cpu = smp_processor_id(); - - if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) { - check_context(next); - cpu_switch_mm(next); - } -} - -#include <asm-generic/mmu_context.h> - -#endif diff --git a/arch/nds32/include/asm/nds32.h b/arch/nds32/include/asm/nds32.h deleted file mode 100644 index 4994f6a9e0a0..000000000000 --- a/arch/nds32/include/asm/nds32.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASM_NDS32_NDS32_H_ -#define _ASM_NDS32_NDS32_H_ - -#include <asm/bitfield.h> -#include <asm/cachectl.h> - -#ifndef __ASSEMBLY__ -#include <linux/init.h> -#include <asm/barrier.h> -#include <nds32_intrinsic.h> - -#ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -#define FP_OFFSET (-3) -#else -#define FP_OFFSET (-2) -#endif -#define LP_OFFSET (-1) - -extern void __init early_trap_init(void); -static inline void GIE_ENABLE(void) -{ - mb(); - __nds32__gie_en(); -} - -static inline void GIE_DISABLE(void) -{ - mb(); - __nds32__gie_dis(); -} - -static inline unsigned long CACHE_SET(unsigned char cache) -{ - - if (cache == ICACHE) - return 64 << ((__nds32__mfsr(NDS32_SR_ICM_CFG) & ICM_CFG_mskISET) >> - ICM_CFG_offISET); - else - return 64 << ((__nds32__mfsr(NDS32_SR_DCM_CFG) & DCM_CFG_mskDSET) >> - DCM_CFG_offDSET); -} - -static inline unsigned long CACHE_WAY(unsigned char cache) -{ - - if (cache == ICACHE) - return 1 + - ((__nds32__mfsr(NDS32_SR_ICM_CFG) & ICM_CFG_mskIWAY) >> ICM_CFG_offIWAY); - else - return 1 + - ((__nds32__mfsr(NDS32_SR_DCM_CFG) & DCM_CFG_mskDWAY) >> DCM_CFG_offDWAY); -} - -static inline unsigned long CACHE_LINE_SIZE(unsigned char cache) -{ - - if (cache == ICACHE) - return 8 << - (((__nds32__mfsr(NDS32_SR_ICM_CFG) & ICM_CFG_mskISZ) >> ICM_CFG_offISZ) - 1); - else - return 8 << - (((__nds32__mfsr(NDS32_SR_DCM_CFG) & DCM_CFG_mskDSZ) >> DCM_CFG_offDSZ) - 1); -} - -#endif /* __ASSEMBLY__ */ - -#define IVB_BASE PHYS_OFFSET /* in user space for intr/exc/trap/break table base, 64KB aligned - * We defined at the start of the physical memory */ - -/* dispatched sub-entry exception handler numbering */ -#define RD_PROT 0 /* read protrection */ -#define WRT_PROT 1 /* write protection */ -#define NOEXEC 2 /* non executable */ -#define PAGE_MODIFY 3 /* page modified */ -#define ACC_BIT 4 /* access bit */ -#define RESVED_PTE 5 /* reserved PTE attribute */ -/* reserved 6 ~ 16 */ - -#endif /* _ASM_NDS32_NDS32_H_ */ diff --git a/arch/nds32/include/asm/nds32_fpu_inst.h b/arch/nds32/include/asm/nds32_fpu_inst.h deleted file mode 100644 index 1e4b86a90a48..000000000000 --- a/arch/nds32/include/asm/nds32_fpu_inst.h +++ /dev/null @@ -1,109 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2005-2018 Andes Technology Corporation */ - -#ifndef __NDS32_FPU_INST_H -#define __NDS32_FPU_INST_H - -#define cop0_op 0x35 - -/* - * COP0 field of opcodes. - */ -#define fs1_op 0x0 -#define fs2_op 0x4 -#define fd1_op 0x8 -#define fd2_op 0xc - -/* - * FS1 opcode. - */ -enum fs1 { - fadds_op, fsubs_op, fcpynss_op, fcpyss_op, - fmadds_op, fmsubs_op, fcmovns_op, fcmovzs_op, - fnmadds_op, fnmsubs_op, - fmuls_op = 0xc, fdivs_op, - fs1_f2op_op = 0xf -}; - -/* - * FS1/F2OP opcode. - */ -enum fs1_f2 { - fs2d_op, fsqrts_op, - fui2s_op = 0x8, fsi2s_op = 0xc, - fs2ui_op = 0x10, fs2ui_z_op = 0x14, - fs2si_op = 0x18, fs2si_z_op = 0x1c -}; - -/* - * FS2 opcode. - */ -enum fs2 { - fcmpeqs_op, fcmpeqs_e_op, fcmplts_op, fcmplts_e_op, - fcmples_op, fcmples_e_op, fcmpuns_op, fcmpuns_e_op -}; - -/* - * FD1 opcode. - */ -enum fd1 { - faddd_op, fsubd_op, fcpynsd_op, fcpysd_op, - fmaddd_op, fmsubd_op, fcmovnd_op, fcmovzd_op, - fnmaddd_op, fnmsubd_op, - fmuld_op = 0xc, fdivd_op, fd1_f2op_op = 0xf -}; - -/* - * FD1/F2OP opcode. - */ -enum fd1_f2 { - fd2s_op, fsqrtd_op, - fui2d_op = 0x8, fsi2d_op = 0xc, - fd2ui_op = 0x10, fd2ui_z_op = 0x14, - fd2si_op = 0x18, fd2si_z_op = 0x1c -}; - -/* - * FD2 opcode. - */ -enum fd2 { - fcmpeqd_op, fcmpeqd_e_op, fcmpltd_op, fcmpltd_e_op, - fcmpled_op, fcmpled_e_op, fcmpund_op, fcmpund_e_op -}; - -#define NDS32Insn(x) x - -#define I_OPCODE_off 25 -#define NDS32Insn_OPCODE(x) (NDS32Insn(x) >> I_OPCODE_off) - -#define I_OPCODE_offRt 20 -#define I_OPCODE_mskRt (0x1fUL << I_OPCODE_offRt) -#define NDS32Insn_OPCODE_Rt(x) \ - ((NDS32Insn(x) & I_OPCODE_mskRt) >> I_OPCODE_offRt) - -#define I_OPCODE_offRa 15 -#define I_OPCODE_mskRa (0x1fUL << I_OPCODE_offRa) -#define NDS32Insn_OPCODE_Ra(x) \ - ((NDS32Insn(x) & I_OPCODE_mskRa) >> I_OPCODE_offRa) - -#define I_OPCODE_offRb 10 -#define I_OPCODE_mskRb (0x1fUL << I_OPCODE_offRb) -#define NDS32Insn_OPCODE_Rb(x) \ - ((NDS32Insn(x) & I_OPCODE_mskRb) >> I_OPCODE_offRb) - -#define I_OPCODE_offbit1014 10 -#define I_OPCODE_mskbit1014 (0x1fUL << I_OPCODE_offbit1014) -#define NDS32Insn_OPCODE_BIT1014(x) \ - ((NDS32Insn(x) & I_OPCODE_mskbit1014) >> I_OPCODE_offbit1014) - -#define I_OPCODE_offbit69 6 -#define I_OPCODE_mskbit69 (0xfUL << I_OPCODE_offbit69) -#define NDS32Insn_OPCODE_BIT69(x) \ - ((NDS32Insn(x) & I_OPCODE_mskbit69) >> I_OPCODE_offbit69) - -#define I_OPCODE_offCOP0 0 -#define I_OPCODE_mskCOP0 (0x3fUL << I_OPCODE_offCOP0) -#define NDS32Insn_OPCODE_COP0(x) \ - ((NDS32Insn(x) & I_OPCODE_mskCOP0) >> I_OPCODE_offCOP0) - -#endif /* __NDS32_FPU_INST_H */ diff --git a/arch/nds32/include/asm/page.h b/arch/nds32/include/asm/page.h deleted file mode 100644 index add33a7f02c8..000000000000 --- a/arch/nds32/include/asm/page.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2005-2017 Andes Technology Corporation - */ - -#ifndef _ASMNDS32_PAGE_H -#define _ASMNDS32_PAGE_H - -#ifdef CONFIG_ANDES_PAGE_SIZE_4KB -#define PAGE_SHIFT 12 -#endif -#ifdef CONFIG_ANDES_PAGE_SIZE_8KB -#define PAGE_SHIFT 13 -#endif -#include <linux/const.h> -#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE-1)) - -#ifdef __KERNEL__ - -#ifndef __ASSEMBLY__ - -struct page; -struct vm_area_struct; -#ifdef CONFIG_CPU_CACHE_ALIASING -extern void copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr, struct vm_area_struct *vma); -extern void clear_user_highpage(struct page *page, unsigned long vaddr); - -void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, - struct page *to); -void clear_user_page(void *addr, unsigned long vaddr, struct page *page); -#define __HAVE_ARCH_COPY_USER_HIGHPAGE -#define clear_user_highpage clear_user_highpage -#else -#define clear_user_page(page, vaddr, pg) clear_page(page) -#define copy_user_page(to, from, vaddr, pg) copy_page(to, from) -#endif - -void clear_page(void *page); -void copy_page(void *to, void *from); - -typedef unsigned long pte_t; -typedef unsigned long pgd_t; -typedef unsigned long pgprot_t; - -#define pte_val(x) (x) -#define pgd_val(x) (x) -#define pgprot_val(x) (x) - -#define __pte(x) (x) -#define __pgd(x) (x) -#define __pgprot(x) (x) - -typedef struct page *pgtable_t; - -#include <asm/memory.h> -#include <asm-generic/getorder.h> - -#endif /* !__ASSEMBLY__ */ - -#endif /* __KERNEL__ */ - -#endif diff --git a/arch/nds32/include/asm/perf_event.h b/arch/nds32/include/asm/perf_event.h deleted file mode 100644 index fcdff02acc14..000000000000 --- a/arch/nds32/include/asm/perf_event.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2008-2018 Andes Technology Corporation */ - -#ifndef __ASM_PERF_EVENT_H -#define __ASM_PERF_EVENT_H - -/* - * This file is request by Perf, - * please refer to tools/perf/design.txt for more details - */ -struct pt_regs; -unsigned long perf_instruction_pointer(struct pt_regs *regs); -unsigned long perf_misc_flags(struct pt_regs *regs); -#define perf_misc_flags(regs) perf_misc_flags(regs) - -#endif diff --git a/arch/nds32/include/asm/pgalloc.h b/arch/nds32/include/asm/pgalloc.h deleted file mode 100644 index a08e1ebca70e..000000000000 --- a/arch/nds32/include/asm/pgalloc.h +++ /dev/null @@ -1,62 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMNDS32_PGALLOC_H -#define _ASMNDS32_PGALLOC_H - -#include <asm/processor.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> -#include <asm/proc-fns.h> - -#define __HAVE_ARCH_PTE_ALLOC_ONE -#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ - -extern pgd_t *pgd_alloc(struct mm_struct *mm); -extern void pgd_free(struct mm_struct *mm, pgd_t * pgd); - -static inline pgtable_t pte_alloc_one(struct mm_struct *mm) -{ - pgtable_t pte; - - pte = __pte_alloc_one(mm, GFP_PGTABLE_USER); - if (pte) - cpu_dcache_wb_page((unsigned long)page_address(pte)); - - return pte; -} - -/* - * Populate the pmdp entry with a pointer to the pte. This pmd is part - * of the mm address space. - * - * Ensure that we always set both PMD entries. - */ -static inline void -pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmdp, pte_t * ptep) -{ - unsigned long pte_ptr = (unsigned long)ptep; - unsigned long pmdval; - - BUG_ON(mm != &init_mm); - - /* - * The pmd must be loaded with the physical - * address of the PTE table - */ - pmdval = __pa(pte_ptr) | _PAGE_KERNEL_TABLE; - set_pmd(pmdp, __pmd(pmdval)); -} - -static inline void -pmd_populate(struct mm_struct *mm, pmd_t * pmdp, pgtable_t ptep) -{ - unsigned long pmdval; - - BUG_ON(mm == &init_mm); - - pmdval = page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE; - set_pmd(pmdp, __pmd(pmdval)); -} - -#endif diff --git a/arch/nds32/include/asm/pgtable.h b/arch/nds32/include/asm/pgtable.h deleted file mode 100644 index 419f984eef70..000000000000 --- a/arch/nds32/include/asm/pgtable.h +++ /dev/null @@ -1,377 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMNDS32_PGTABLE_H -#define _ASMNDS32_PGTABLE_H - -#include <asm-generic/pgtable-nopmd.h> -#include <linux/sizes.h> - -#include <asm/memory.h> -#include <asm/nds32.h> -#ifndef __ASSEMBLY__ -#include <asm/fixmap.h> -#include <nds32_intrinsic.h> -#endif - -#ifdef CONFIG_ANDES_PAGE_SIZE_4KB -#define PGDIR_SHIFT 22 -#define PTRS_PER_PGD 1024 -#define PTRS_PER_PTE 1024 -#endif - -#ifdef CONFIG_ANDES_PAGE_SIZE_8KB -#define PGDIR_SHIFT 24 -#define PTRS_PER_PGD 256 -#define PTRS_PER_PTE 2048 -#endif - -#ifndef __ASSEMBLY__ -extern void __pte_error(const char *file, int line, unsigned long val); -extern void __pgd_error(const char *file, int line, unsigned long val); - -#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte)) -#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) -#endif /* !__ASSEMBLY__ */ - -#define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) -#define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) - -/* - * This is the lowest virtual address we can permit any user space - * mapping to be mapped at. This is particularly important for - * non-high vector CPUs. - */ -#define FIRST_USER_ADDRESS 0x8000 - -#ifdef CONFIG_HIGHMEM -#define CONSISTENT_BASE ((PKMAP_BASE) - (SZ_2M)) -#define CONSISTENT_END (PKMAP_BASE) -#else -#define CONSISTENT_BASE (FIXADDR_START - SZ_2M) -#define CONSISTENT_END (FIXADDR_START) -#endif -#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) - -#ifdef CONFIG_HIGHMEM -#ifndef __ASSEMBLY__ -#include <asm/highmem.h> -#endif -#endif - -#define VMALLOC_RESERVE SZ_128M -#define VMALLOC_END (CONSISTENT_BASE - PAGE_SIZE) -#define VMALLOC_START ((VMALLOC_END) - VMALLOC_RESERVE) -#define VMALLOC_VMADDR(x) ((unsigned long)(x)) -#define MAXMEM __pa(VMALLOC_START) -#define MAXMEM_PFN PFN_DOWN(MAXMEM) - -#define FIRST_USER_PGD_NR 0 -#define USER_PTRS_PER_PGD ((TASK_SIZE/PGDIR_SIZE) + FIRST_USER_PGD_NR) - -/* L2 PTE */ -#define _PAGE_V (1UL << 0) - -#define _PAGE_M_XKRW (0UL << 1) -#define _PAGE_M_UR_KR (1UL << 1) -#define _PAGE_M_UR_KRW (2UL << 1) -#define _PAGE_M_URW_KRW (3UL << 1) -#define _PAGE_M_KR (5UL << 1) -#define _PAGE_M_KRW (7UL << 1) - -#define _PAGE_D (1UL << 4) -#define _PAGE_E (1UL << 5) -#define _PAGE_A (1UL << 6) -#define _PAGE_G (1UL << 7) - -#define _PAGE_C_DEV (0UL << 8) -#define _PAGE_C_DEV_WB (1UL << 8) -#define _PAGE_C_MEM (2UL << 8) -#define _PAGE_C_MEM_SHRD_WB (4UL << 8) -#define _PAGE_C_MEM_SHRD_WT (5UL << 8) -#define _PAGE_C_MEM_WB (6UL << 8) -#define _PAGE_C_MEM_WT (7UL << 8) - -#define _PAGE_L (1UL << 11) - -#define _HAVE_PAGE_L (_PAGE_L) -#define _PAGE_FILE (1UL << 1) -#define _PAGE_YOUNG 0 -#define _PAGE_M_MASK _PAGE_M_KRW -#define _PAGE_C_MASK _PAGE_C_MEM_WT - -#ifdef CONFIG_SMP -#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH -#define _PAGE_CACHE_SHRD _PAGE_C_MEM_SHRD_WT -#else -#define _PAGE_CACHE_SHRD _PAGE_C_MEM_SHRD_WB -#endif -#else -#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH -#define _PAGE_CACHE_SHRD _PAGE_C_MEM_WT -#else -#define _PAGE_CACHE_SHRD _PAGE_C_MEM_WB -#endif -#endif - -#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH -#define _PAGE_CACHE _PAGE_C_MEM_WT -#else -#define _PAGE_CACHE _PAGE_C_MEM_WB -#endif - -#define _PAGE_IOREMAP \ - (_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_G | _PAGE_C_DEV) - -/* - * + Level 1 descriptor (PMD) - */ -#define PMD_TYPE_TABLE 0 - -#ifndef __ASSEMBLY__ - -#define _PAGE_USER_TABLE PMD_TYPE_TABLE -#define _PAGE_KERNEL_TABLE PMD_TYPE_TABLE - -#define PAGE_EXEC __pgprot(_PAGE_V | _PAGE_M_XKRW | _PAGE_E) -#define PAGE_NONE __pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_A) -#define PAGE_READ __pgprot(_PAGE_V | _PAGE_M_UR_KR) -#define PAGE_RDWR __pgprot(_PAGE_V | _PAGE_M_URW_KRW | _PAGE_D) -#define PAGE_COPY __pgprot(_PAGE_V | _PAGE_M_UR_KR) - -#define PAGE_UXKRWX_V1 __pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD) -#define PAGE_UXKRWX_V2 __pgprot(_PAGE_V | _PAGE_M_XKRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD) -#define PAGE_URXKRWX_V2 __pgprot(_PAGE_V | _PAGE_M_UR_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD) -#define PAGE_CACHE_L1 __pgprot(_HAVE_PAGE_L | _PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE) -#define PAGE_MEMORY __pgprot(_HAVE_PAGE_L | _PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD) -#define PAGE_KERNEL __pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD) -#define PAGE_SHARED __pgprot(_PAGE_V | _PAGE_M_URW_KRW | _PAGE_D | _PAGE_CACHE_SHRD) -#define PAGE_DEVICE __pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_G | _PAGE_C_DEV) -#endif /* __ASSEMBLY__ */ - -/* xwr */ -#define __P000 (PAGE_NONE | _PAGE_CACHE_SHRD) -#define __P001 (PAGE_READ | _PAGE_CACHE_SHRD) -#define __P010 (PAGE_COPY | _PAGE_CACHE_SHRD) -#define __P011 (PAGE_COPY | _PAGE_CACHE_SHRD) -#define __P100 (PAGE_EXEC | _PAGE_CACHE_SHRD) -#define __P101 (PAGE_READ | _PAGE_E | _PAGE_CACHE_SHRD) -#define __P110 (PAGE_COPY | _PAGE_E | _PAGE_CACHE_SHRD) -#define __P111 (PAGE_COPY | _PAGE_E | _PAGE_CACHE_SHRD) - -#define __S000 (PAGE_NONE | _PAGE_CACHE_SHRD) -#define __S001 (PAGE_READ | _PAGE_CACHE_SHRD) -#define __S010 (PAGE_RDWR | _PAGE_CACHE_SHRD) -#define __S011 (PAGE_RDWR | _PAGE_CACHE_SHRD) -#define __S100 (PAGE_EXEC | _PAGE_CACHE_SHRD) -#define __S101 (PAGE_READ | _PAGE_E | _PAGE_CACHE_SHRD) -#define __S110 (PAGE_RDWR | _PAGE_E | _PAGE_CACHE_SHRD) -#define __S111 (PAGE_RDWR | _PAGE_E | _PAGE_CACHE_SHRD) - -#ifndef __ASSEMBLY__ -/* - * ZERO_PAGE is a global shared page that is always zero: used - * for zero-mapped memory areas etc.. - */ -extern struct page *empty_zero_page; -extern void paging_init(void); -#define ZERO_PAGE(vaddr) (empty_zero_page) - -#define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) -#define pfn_pte(pfn,prot) (__pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))) - -#define pte_none(pte) !(pte_val(pte)) -#define pte_clear(mm,addr,ptep) set_pte_at((mm),(addr),(ptep), __pte(0)) -#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) - -static unsigned long pmd_page_vaddr(pmd_t pmd) -{ - return ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)); -} - -#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) -/* - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - */ -static inline void set_pmd(pmd_t * pmdp, pmd_t pmd) -{ - - *pmdp = pmd; -#if !defined(CONFIG_CPU_DCACHE_DISABLE) && !defined(CONFIG_CPU_DCACHE_WRITETHROUGH) - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (pmdp):"memory"); - __nds32__msync_all(); - __nds32__dsb(); -#endif -} - -/* - * Set a PTE and flush it out - */ -static inline void set_pte(pte_t * ptep, pte_t pte) -{ - - *ptep = pte; -#if !defined(CONFIG_CPU_DCACHE_DISABLE) && !defined(CONFIG_CPU_DCACHE_WRITETHROUGH) - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (ptep):"memory"); - __nds32__msync_all(); - __nds32__dsb(); -#endif -} - -/* - * The following only work if pte_present() is true. - * Undefined behaviour if not.. - */ - -/* - * pte_write: this page is writeable for user mode - * pte_read: this page is readable for user mode - * pte_kernel_write: this page is writeable for kernel mode - * - * We don't have pte_kernel_read because kernel always can read. - * - * */ - -#define pte_present(pte) (pte_val(pte) & _PAGE_V) -#define pte_write(pte) ((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_URW_KRW) -#define pte_read(pte) (((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_UR_KR) || \ - ((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_UR_KRW) || \ - ((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_URW_KRW)) -#define pte_kernel_write(pte) (((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_URW_KRW) || \ - ((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_UR_KRW) || \ - ((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_KRW) || \ - (((pte_val(pte) & _PAGE_M_MASK) == _PAGE_M_XKRW) && pte_exec(pte))) -#define pte_exec(pte) (pte_val(pte) & _PAGE_E) -#define pte_dirty(pte) (pte_val(pte) & _PAGE_D) -#define pte_young(pte) (pte_val(pte) & _PAGE_YOUNG) - -/* - * The following only works if pte_present() is not true. - */ -#define pte_file(pte) (pte_val(pte) & _PAGE_FILE) -#define pte_to_pgoff(x) (pte_val(x) >> 2) -#define pgoff_to_pte(x) __pte(((x) << 2) | _PAGE_FILE) - -#define PTE_FILE_MAX_BITS 29 - -#define PTE_BIT_FUNC(fn,op) \ -static inline pte_t pte_##fn(pte_t pte) { pte_val(pte) op; return pte; } - -static inline pte_t pte_wrprotect(pte_t pte) -{ - pte_val(pte) = pte_val(pte) & ~_PAGE_M_MASK; - pte_val(pte) = pte_val(pte) | _PAGE_M_UR_KR; - return pte; -} - -static inline pte_t pte_mkwrite(pte_t pte) -{ - pte_val(pte) = pte_val(pte) & ~_PAGE_M_MASK; - pte_val(pte) = pte_val(pte) | _PAGE_M_URW_KRW; - return pte; -} - -PTE_BIT_FUNC(exprotect, &=~_PAGE_E); -PTE_BIT_FUNC(mkexec, |=_PAGE_E); -PTE_BIT_FUNC(mkclean, &=~_PAGE_D); -PTE_BIT_FUNC(mkdirty, |=_PAGE_D); -PTE_BIT_FUNC(mkold, &=~_PAGE_YOUNG); -PTE_BIT_FUNC(mkyoung, |=_PAGE_YOUNG); - -/* - * Mark the prot value as uncacheable and unbufferable. - */ -#define pgprot_noncached(prot) __pgprot((pgprot_val(prot)&~_PAGE_C_MASK) | _PAGE_C_DEV) -#define pgprot_writecombine(prot) __pgprot((pgprot_val(prot)&~_PAGE_C_MASK) | _PAGE_C_DEV_WB) - -#define pmd_none(pmd) (pmd_val(pmd)&0x1) -#define pmd_present(pmd) (!pmd_none(pmd)) -#define pmd_bad(pmd) pmd_none(pmd) - -#define copy_pmd(pmdpd,pmdps) set_pmd((pmdpd), *(pmdps)) -#define pmd_clear(pmdp) set_pmd((pmdp), __pmd(1)) - -static inline pmd_t __mk_pmd(pte_t * ptep, unsigned long prot) -{ - unsigned long ptr = (unsigned long)ptep; - pmd_t pmd; - - /* - * The pmd must be loaded with the physical - * address of the PTE table - */ - - pmd_val(pmd) = __virt_to_phys(ptr) | prot; - return pmd; -} - -#define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd))) - -/* - * Permanent address of a page. We never have highmem, so this is trivial. - */ -#define pages_to_mb(x) ((x) >> (20 - PAGE_SHIFT)) - -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot) - -/* - * The "pgd_xxx()" functions here are trivial for a folded two-level - * setup: the pgd is never bad, and a pmd always exists (as it's folded - * into the pgd entry) - */ -#define pgd_none(pgd) (0) -#define pgd_bad(pgd) (0) -#define pgd_present(pgd) (1) -#define pgd_clear(pgdp) do { } while (0) - -#define page_pte_prot(page,prot) mk_pte(page, prot) -#define page_pte(page) mk_pte(page, __pgprot(0)) -/* - * L1PTE = $mr1 + ((virt >> PMD_SHIFT) << 2); - * L2PTE = (((virt >> PAGE_SHIFT) & (PTRS_PER_PTE -1 )) << 2); - * PPN = (phys & 0xfffff000); - * -*/ - -static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) -{ - const unsigned long mask = 0xfff; - pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); - return pte; -} - -extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; - -/* Encode and decode a swap entry. - * - * We support up to 32GB of swap on 4k machines - */ -#define __swp_type(x) (((x).val >> 2) & 0x7f) -#define __swp_offset(x) ((x).val >> 9) -#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << 2) | ((offset) << 9) }) -#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) -#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) - -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -#define kern_addr_valid(addr) (1) - -/* - * We provide our own arch_get_unmapped_area to cope with VIPT caches. - */ -#define HAVE_ARCH_UNMAPPED_AREA - -/* - * remap a physical address `phys' of size `size' with page protection `prot' - * into virtual address `from' - */ - -#endif /* !__ASSEMBLY__ */ - -#endif /* _ASMNDS32_PGTABLE_H */ diff --git a/arch/nds32/include/asm/pmu.h b/arch/nds32/include/asm/pmu.h deleted file mode 100644 index e1ac0b0b8bcf..000000000000 --- a/arch/nds32/include/asm/pmu.h +++ /dev/null @@ -1,386 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2008-2018 Andes Technology Corporation */ - -#ifndef __ASM_PMU_H -#define __ASM_PMU_H - -#include <linux/interrupt.h> -#include <linux/perf_event.h> -#include <asm/unistd.h> -#include <asm/bitfield.h> - -/* Has special meaning for perf core implementation */ -#define HW_OP_UNSUPPORTED 0x0 -#define C(_x) PERF_COUNT_HW_CACHE_##_x -#define CACHE_OP_UNSUPPORTED 0x0 - -/* Enough for both software and hardware defined events */ -#define SOFTWARE_EVENT_MASK 0xFF - -#define PFM_OFFSET_MAGIC_0 2 /* DO NOT START FROM 0 */ -#define PFM_OFFSET_MAGIC_1 (PFM_OFFSET_MAGIC_0 + 36) -#define PFM_OFFSET_MAGIC_2 (PFM_OFFSET_MAGIC_1 + 36) - -enum { PFMC0, PFMC1, PFMC2, MAX_COUNTERS }; - -u32 PFM_CTL_OVF[3] = { PFM_CTL_mskOVF0, PFM_CTL_mskOVF1, - PFM_CTL_mskOVF2 }; -u32 PFM_CTL_EN[3] = { PFM_CTL_mskEN0, PFM_CTL_mskEN1, - PFM_CTL_mskEN2 }; -u32 PFM_CTL_OFFSEL[3] = { PFM_CTL_offSEL0, PFM_CTL_offSEL1, - PFM_CTL_offSEL2 }; -u32 PFM_CTL_IE[3] = { PFM_CTL_mskIE0, PFM_CTL_mskIE1, PFM_CTL_mskIE2 }; -u32 PFM_CTL_KS[3] = { PFM_CTL_mskKS0, PFM_CTL_mskKS1, PFM_CTL_mskKS2 }; -u32 PFM_CTL_KU[3] = { PFM_CTL_mskKU0, PFM_CTL_mskKU1, PFM_CTL_mskKU2 }; -u32 PFM_CTL_SEL[3] = { PFM_CTL_mskSEL0, PFM_CTL_mskSEL1, PFM_CTL_mskSEL2 }; -/* - * Perf Events' indices - */ -#define NDS32_IDX_CYCLE_COUNTER 0 -#define NDS32_IDX_COUNTER0 1 -#define NDS32_IDX_COUNTER1 2 - -/* The events for a given PMU register set. */ -struct pmu_hw_events { - /* - * The events that are active on the PMU for the given index. - */ - struct perf_event *events[MAX_COUNTERS]; - - /* - * A 1 bit for an index indicates that the counter is being used for - * an event. A 0 means that the counter can be used. - */ - unsigned long used_mask[BITS_TO_LONGS(MAX_COUNTERS)]; - - /* - * Hardware lock to serialize accesses to PMU registers. Needed for the - * read/modify/write sequences. - */ - raw_spinlock_t pmu_lock; -}; - -struct nds32_pmu { - struct pmu pmu; - cpumask_t active_irqs; - char *name; - irqreturn_t (*handle_irq)(int irq_num, void *dev); - void (*enable)(struct perf_event *event); - void (*disable)(struct perf_event *event); - int (*get_event_idx)(struct pmu_hw_events *hw_events, - struct perf_event *event); - int (*set_event_filter)(struct hw_perf_event *evt, - struct perf_event_attr *attr); - u32 (*read_counter)(struct perf_event *event); - void (*write_counter)(struct perf_event *event, u32 val); - void (*start)(struct nds32_pmu *nds32_pmu); - void (*stop)(struct nds32_pmu *nds32_pmu); - void (*reset)(void *data); - int (*request_irq)(struct nds32_pmu *nds32_pmu, irq_handler_t handler); - void (*free_irq)(struct nds32_pmu *nds32_pmu); - int (*map_event)(struct perf_event *event); - int num_events; - atomic_t active_events; - u64 max_period; - struct platform_device *plat_device; - struct pmu_hw_events *(*get_hw_events)(void); -}; - -#define to_nds32_pmu(p) (container_of(p, struct nds32_pmu, pmu)) - -int nds32_pmu_register(struct nds32_pmu *nds32_pmu, int type); - -u64 nds32_pmu_event_update(struct perf_event *event); - -int nds32_pmu_event_set_period(struct perf_event *event); - -/* - * Common NDS32 SPAv3 event types - * - * Note: An implementation may not be able to count all of these events - * but the encodings are considered to be `reserved' in the case that - * they are not available. - * - * SEL_TOTAL_CYCLES will add an offset is due to ZERO is defined as - * NOT_SUPPORTED EVENT mapping in generic perf code. - * You will need to deal it in the event writing implementation. - */ -enum spav3_counter_0_perf_types { - SPAV3_0_SEL_BASE = -1 + PFM_OFFSET_MAGIC_0, /* counting symbol */ - SPAV3_0_SEL_TOTAL_CYCLES = 0 + PFM_OFFSET_MAGIC_0, - SPAV3_0_SEL_COMPLETED_INSTRUCTION = 1 + PFM_OFFSET_MAGIC_0, - SPAV3_0_SEL_LAST /* counting symbol */ -}; - -enum spav3_counter_1_perf_types { - SPAV3_1_SEL_BASE = -1 + PFM_OFFSET_MAGIC_1, /* counting symbol */ - SPAV3_1_SEL_TOTAL_CYCLES = 0 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_COMPLETED_INSTRUCTION = 1 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_CONDITIONAL_BRANCH = 2 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_TAKEN_CONDITIONAL_BRANCH = 3 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_PREFETCH_INSTRUCTION = 4 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_RET_INST = 5 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_JR_INST = 6 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_JAL_JRAL_INST = 7 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_NOP_INST = 8 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_SCW_INST = 9 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_ISB_DSB_INST = 10 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_CCTL_INST = 11 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_TAKEN_INTERRUPTS = 12 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_LOADS_COMPLETED = 13 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_UITLB_ACCESS = 14 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_UDTLB_ACCESS = 15 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_MTLB_ACCESS = 16 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_CODE_CACHE_ACCESS = 17 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_DATA_DEPENDENCY_STALL_CYCLES = 18 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_DATA_CACHE_MISS_STALL_CYCLES = 19 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_DATA_CACHE_ACCESS = 20 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_DATA_CACHE_MISS = 21 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_LOAD_DATA_CACHE_ACCESS = 22 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_STORE_DATA_CACHE_ACCESS = 23 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_ILM_ACCESS = 24 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_LSU_BIU_CYCLES = 25 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_HPTWK_BIU_CYCLES = 26 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_DMA_BIU_CYCLES = 27 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_CODE_CACHE_FILL_BIU_CYCLES = 28 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_LEGAL_UNALIGN_DCACHE_ACCESS = 29 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_PUSH25 = 30 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_SYSCALLS_INST = 31 + PFM_OFFSET_MAGIC_1, - SPAV3_1_SEL_LAST /* counting symbol */ -}; - -enum spav3_counter_2_perf_types { - SPAV3_2_SEL_BASE = -1 + PFM_OFFSET_MAGIC_2, /* counting symbol */ - SPAV3_2_SEL_TOTAL_CYCLES = 0 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_COMPLETED_INSTRUCTION = 1 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_CONDITIONAL_BRANCH_MISPREDICT = 2 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_TAKEN_CONDITIONAL_BRANCH_MISPREDICT = - 3 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_PREFETCH_INSTRUCTION_CACHE_HIT = 4 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_RET_MISPREDICT = 5 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_IMMEDIATE_J_INST = 6 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_MULTIPLY_INST = 7 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_16_BIT_INST = 8 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_FAILED_SCW_INST = 9 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_LD_AFTER_ST_CONFLICT_REPLAYS = 10 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_TAKEN_EXCEPTIONS = 12 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_STORES_COMPLETED = 13 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_UITLB_MISS = 14 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_UDTLB_MISS = 15 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_MTLB_MISS = 16 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_CODE_CACHE_MISS = 17 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_EMPTY_INST_QUEUE_STALL_CYCLES = 18 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_DATA_WRITE_BACK = 19 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_DATA_CACHE_MISS = 21 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_LOAD_DATA_CACHE_MISS = 22 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_STORE_DATA_CACHE_MISS = 23 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_DLM_ACCESS = 24 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_LSU_BIU_REQUEST = 25 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_HPTWK_BIU_REQUEST = 26 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_DMA_BIU_REQUEST = 27 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_CODE_CACHE_FILL_BIU_REQUEST = 28 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_EXTERNAL_EVENTS = 29 + PFM_OFFSET_MAGIC_2, - SPAV3_1_SEL_POP25 = 30 + PFM_OFFSET_MAGIC_2, - SPAV3_2_SEL_LAST /* counting symbol */ -}; - -/* Get converted event counter index */ -static inline int get_converted_event_idx(unsigned long event) -{ - int idx; - - if ((event) > SPAV3_0_SEL_BASE && event < SPAV3_0_SEL_LAST) { - idx = 0; - } else if ((event) > SPAV3_1_SEL_BASE && event < SPAV3_1_SEL_LAST) { - idx = 1; - } else if ((event) > SPAV3_2_SEL_BASE && event < SPAV3_2_SEL_LAST) { - idx = 2; - } else { - pr_err("GET_CONVERTED_EVENT_IDX PFM counter range error\n"); - return -EPERM; - } - - return idx; -} - -/* Get converted hardware event number */ -static inline u32 get_converted_evet_hw_num(u32 event) -{ - if (event > SPAV3_0_SEL_BASE && event < SPAV3_0_SEL_LAST) - event -= PFM_OFFSET_MAGIC_0; - else if (event > SPAV3_1_SEL_BASE && event < SPAV3_1_SEL_LAST) - event -= PFM_OFFSET_MAGIC_1; - else if (event > SPAV3_2_SEL_BASE && event < SPAV3_2_SEL_LAST) - event -= PFM_OFFSET_MAGIC_2; - else if (event != 0) - pr_err("GET_CONVERTED_EVENT_HW_NUM PFM counter range error\n"); - - return event; -} - -/* - * NDS32 HW events mapping - * - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned int nds32_pfm_perf_map[PERF_COUNT_HW_MAX] = { - [PERF_COUNT_HW_CPU_CYCLES] = SPAV3_0_SEL_TOTAL_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = SPAV3_1_SEL_COMPLETED_INSTRUCTION, - [PERF_COUNT_HW_CACHE_REFERENCES] = SPAV3_1_SEL_DATA_CACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = SPAV3_2_SEL_DATA_CACHE_MISS, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_BRANCH_MISSES] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_REF_CPU_CYCLES] = HW_OP_UNSUPPORTED -}; - -static const unsigned int nds32_pfm_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - [C(L1D)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_LOAD_DATA_CACHE_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_LOAD_DATA_CACHE_MISS, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_STORE_DATA_CACHE_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_STORE_DATA_CACHE_MISS, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - }, - [C(L1I)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_CODE_CACHE_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_CODE_CACHE_MISS, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_CODE_CACHE_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_CODE_CACHE_MISS, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - /* TODO: L2CC */ - [C(LL)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - /* NDS32 PMU does not support TLB read/write hit/miss, - * However, it can count access/miss, which mixed with read and write. - * Therefore, only READ counter will use it. - * We do as possible as we can. - */ - [C(DTLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_UDTLB_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_UDTLB_MISS, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - }, - [C(ITLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = - SPAV3_1_SEL_UITLB_ACCESS, - [C(RESULT_MISS)] = - SPAV3_2_SEL_UITLB_MISS, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - }, - [C(BPU)] = { /* What is BPU? */ - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - }, - [C(NODE)] = { /* What is NODE? */ - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = - CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = - CACHE_OP_UNSUPPORTED, - }, - }, -}; - -int nds32_pmu_map_event(struct perf_event *event, - const unsigned int (*event_map)[PERF_COUNT_HW_MAX], - const unsigned int (*cache_map)[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask); - -#endif /* __ASM_PMU_H */ diff --git a/arch/nds32/include/asm/proc-fns.h b/arch/nds32/include/asm/proc-fns.h deleted file mode 100644 index 27c617fa77af..000000000000 --- a/arch/nds32/include/asm/proc-fns.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_PROCFNS_H__ -#define __NDS32_PROCFNS_H__ - -#ifdef __KERNEL__ -#include <asm/page.h> - -struct mm_struct; -struct vm_area_struct; -extern void cpu_proc_init(void); -extern void cpu_proc_fin(void); -extern void cpu_do_idle(void); -extern void cpu_reset(unsigned long reset); -extern void cpu_switch_mm(struct mm_struct *mm); - -extern void cpu_dcache_inval_all(void); -extern void cpu_dcache_wbinval_all(void); -extern void cpu_dcache_inval_page(unsigned long page); -extern void cpu_dcache_wb_page(unsigned long page); -extern void cpu_dcache_wbinval_page(unsigned long page); -extern void cpu_dcache_inval_range(unsigned long start, unsigned long end); -extern void cpu_dcache_wb_range(unsigned long start, unsigned long end); -extern void cpu_dcache_wbinval_range(unsigned long start, unsigned long end); - -extern void cpu_icache_inval_all(void); -extern void cpu_icache_inval_page(unsigned long page); -extern void cpu_icache_inval_range(unsigned long start, unsigned long end); - -extern void cpu_cache_wbinval_page(unsigned long page, int flushi); -extern void cpu_cache_wbinval_range(unsigned long start, - unsigned long end, int flushi); -extern void cpu_cache_wbinval_range_check(struct vm_area_struct *vma, - unsigned long start, - unsigned long end, bool flushi, - bool wbd); - -extern void cpu_dma_wb_range(unsigned long start, unsigned long end); -extern void cpu_dma_inval_range(unsigned long start, unsigned long end); -extern void cpu_dma_wbinval_range(unsigned long start, unsigned long end); - -#endif /* __KERNEL__ */ -#endif /* __NDS32_PROCFNS_H__ */ diff --git a/arch/nds32/include/asm/processor.h b/arch/nds32/include/asm/processor.h deleted file mode 100644 index e6bfc74972bb..000000000000 --- a/arch/nds32/include/asm/processor.h +++ /dev/null @@ -1,104 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_PROCESSOR_H -#define __ASM_NDS32_PROCESSOR_H - -#ifdef __KERNEL__ - -#include <asm/ptrace.h> -#include <asm/types.h> -#include <asm/sigcontext.h> - -#define KERNEL_STACK_SIZE PAGE_SIZE -#define STACK_TOP TASK_SIZE -#define STACK_TOP_MAX TASK_SIZE - -struct cpu_context { - unsigned long r6; - unsigned long r7; - unsigned long r8; - unsigned long r9; - unsigned long r10; - unsigned long r11; - unsigned long r12; - unsigned long r13; - unsigned long r14; - unsigned long fp; - unsigned long pc; - unsigned long sp; -}; - -struct thread_struct { - struct cpu_context cpu_context; /* cpu context */ - /* fault info */ - unsigned long address; - unsigned long trap_no; - unsigned long error_code; - - struct fpu_struct fpu; -}; - -#define INIT_THREAD { } - -#ifdef __NDS32_EB__ -#define PSW_DE PSW_mskBE -#else -#define PSW_DE 0x0 -#endif - -#ifdef CONFIG_WBNA -#define PSW_valWBNA PSW_mskWBNA -#else -#define PSW_valWBNA 0x0 -#endif - -#ifdef CONFIG_HWZOL -#define PSW_valINIT (PSW_CPL_ANY | PSW_mskAEN | PSW_valWBNA | PSW_mskDT | PSW_mskIT | PSW_DE | PSW_mskGIE) -#else -#define PSW_valINIT (PSW_CPL_ANY | PSW_valWBNA | PSW_mskDT | PSW_mskIT | PSW_DE | PSW_mskGIE) -#endif - -#define start_thread(regs,pc,stack) \ -({ \ - memzero(regs, sizeof(struct pt_regs)); \ - forget_syscall(regs); \ - regs->ipsw = PSW_valINIT; \ - regs->ir0 = (PSW_CPL_ANY | PSW_valWBNA | PSW_mskDT | PSW_mskIT | PSW_DE | PSW_SYSTEM | PSW_INTL_1); \ - regs->ipc = pc; \ - regs->sp = stack; \ -}) - -/* Forward declaration, a strange C thing */ -struct task_struct; - -/* Free all resources held by a thread. */ -#define release_thread(thread) do { } while(0) -#if IS_ENABLED(CONFIG_FPU) -#if !IS_ENABLED(CONFIG_UNLAZU_FPU) -extern struct task_struct *last_task_used_math; -#endif -#endif - -/* Prepare to copy thread state - unlazy all lazy status */ -#define prepare_to_copy(tsk) do { } while (0) - -unsigned long __get_wchan(struct task_struct *p); - -#define cpu_relax() barrier() - -#define task_pt_regs(task) \ - ((struct pt_regs *) (task_stack_page(task) + THREAD_SIZE \ - - 8) - 1) - -/* - * Create a new kernel thread - */ -extern int kernel_thread(int (*fn) (void *), void *arg, unsigned long flags); - -#define KSTK_EIP(tsk) instruction_pointer(task_pt_regs(tsk)) -#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) - -#endif - -#endif /* __ASM_NDS32_PROCESSOR_H */ diff --git a/arch/nds32/include/asm/ptrace.h b/arch/nds32/include/asm/ptrace.h deleted file mode 100644 index 919ee223620c..000000000000 --- a/arch/nds32/include/asm/ptrace.h +++ /dev/null @@ -1,77 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_PTRACE_H -#define __ASM_NDS32_PTRACE_H - -#include <uapi/asm/ptrace.h> - -/* - * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing - * a syscall -- i.e., its most recent entry into the kernel from - * userspace was not via syscall, or otherwise a tracer cancelled the - * syscall. - * - * This must have the value -1, for ABI compatibility with ptrace etc. - */ -#define NO_SYSCALL (-1) -#ifndef __ASSEMBLY__ -#include <linux/types.h> - -struct pt_regs { - union { - struct user_pt_regs user_regs; - struct { - long uregs[26]; - long fp; - long gp; - long lp; - long sp; - long ipc; -#if defined(CONFIG_HWZOL) - long lb; - long le; - long lc; -#else - long dummy[3]; -#endif - long syscallno; - }; - }; - long orig_r0; - long ir0; - long ipsw; - long pipsw; - long pipc; - long pp0; - long pp1; - long fucop_ctl; - long osp; -}; - -static inline bool in_syscall(struct pt_regs const *regs) -{ - return regs->syscallno != NO_SYSCALL; -} - -static inline void forget_syscall(struct pt_regs *regs) -{ - regs->syscallno = NO_SYSCALL; -} -static inline unsigned long regs_return_value(struct pt_regs *regs) -{ - return regs->uregs[0]; -} -extern void show_regs(struct pt_regs *); -/* Avoid circular header include via sched.h */ -struct task_struct; - -#define arch_has_single_step() (1) -#define user_mode(regs) (((regs)->ipsw & PSW_mskPOM) == 0) -#define interrupts_enabled(regs) (!!((regs)->ipsw & PSW_mskGIE)) -#define user_stack_pointer(regs) ((regs)->sp) -#define instruction_pointer(regs) ((regs)->ipc) -#define profile_pc(regs) instruction_pointer(regs) - -#endif /* __ASSEMBLY__ */ -#endif diff --git a/arch/nds32/include/asm/sfp-machine.h b/arch/nds32/include/asm/sfp-machine.h deleted file mode 100644 index b1a5caa332b5..000000000000 --- a/arch/nds32/include/asm/sfp-machine.h +++ /dev/null @@ -1,158 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2005-2018 Andes Technology Corporation */ - -#include <asm/bitfield.h> - -#define _FP_W_TYPE_SIZE 32 -#define _FP_W_TYPE unsigned long -#define _FP_WS_TYPE signed long -#define _FP_I_TYPE long - -#define __ll_B ((UWtype) 1 << (W_TYPE_SIZE / 2)) -#define __ll_lowpart(t) ((UWtype) (t) & (__ll_B - 1)) -#define __ll_highpart(t) ((UWtype) (t) >> (W_TYPE_SIZE / 2)) - -#define _FP_MUL_MEAT_S(R, X, Y) \ - _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_S, R, X, Y, umul_ppmm) -#define _FP_MUL_MEAT_D(R, X, Y) \ - _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_D, R, X, Y, umul_ppmm) -#define _FP_MUL_MEAT_Q(R, X, Y) \ - _FP_MUL_MEAT_4_wide(_FP_WFRACBITS_Q, R, X, Y, umul_ppmm) - -#define _FP_MUL_MEAT_DW_S(R, X, Y) \ - _FP_MUL_MEAT_DW_1_wide(_FP_WFRACBITS_S, R, X, Y, umul_ppmm) -#define _FP_MUL_MEAT_DW_D(R, X, Y) \ - _FP_MUL_MEAT_DW_2_wide(_FP_WFRACBITS_D, R, X, Y, umul_ppmm) - -#define _FP_DIV_MEAT_S(R, X, Y) _FP_DIV_MEAT_1_udiv_norm(S, R, X, Y) -#define _FP_DIV_MEAT_D(R, X, Y) _FP_DIV_MEAT_2_udiv(D, R, X, Y) - -#define _FP_NANFRAC_S ((_FP_QNANBIT_S << 1) - 1) -#define _FP_NANFRAC_D ((_FP_QNANBIT_D << 1) - 1), -1 -#define _FP_NANFRAC_Q ((_FP_QNANBIT_Q << 1) - 1), -1, -1, -1 -#define _FP_NANSIGN_S 0 -#define _FP_NANSIGN_D 0 -#define _FP_NANSIGN_Q 0 - -#define _FP_KEEPNANFRACP 1 -#define _FP_QNANNEGATEDP 0 - -#define _FP_CHOOSENAN(fs, wc, R, X, Y, OP) \ -do { \ - if ((_FP_FRAC_HIGH_RAW_##fs(X) & _FP_QNANBIT_##fs) \ - && !(_FP_FRAC_HIGH_RAW_##fs(Y) & _FP_QNANBIT_##fs)) { \ - R##_s = Y##_s; \ - _FP_FRAC_COPY_##wc(R, Y); \ - } else { \ - R##_s = X##_s; \ - _FP_FRAC_COPY_##wc(R, X); \ - } \ - R##_c = FP_CLS_NAN; \ -} while (0) - -#define __FPU_FPCSR (current->thread.fpu.fpcsr) - -/* Obtain the current rounding mode. */ -#define FP_ROUNDMODE \ -({ \ - __FPU_FPCSR & FPCSR_mskRM; \ -}) - -#define FP_RND_NEAREST 0 -#define FP_RND_PINF 1 -#define FP_RND_MINF 2 -#define FP_RND_ZERO 3 - -#define FP_EX_INVALID FPCSR_mskIVO -#define FP_EX_DIVZERO FPCSR_mskDBZ -#define FP_EX_OVERFLOW FPCSR_mskOVF -#define FP_EX_UNDERFLOW FPCSR_mskUDF -#define FP_EX_INEXACT FPCSR_mskIEX - -#define SF_CEQ 2 -#define SF_CLT 1 -#define SF_CGT 3 -#define SF_CUN 4 - -#include <asm/byteorder.h> - -#ifdef __BIG_ENDIAN__ -#define __BYTE_ORDER __BIG_ENDIAN -#define __LITTLE_ENDIAN 0 -#else -#define __BYTE_ORDER __LITTLE_ENDIAN -#define __BIG_ENDIAN 0 -#endif - -#define abort() do { } while (0) -#define umul_ppmm(w1, w0, u, v) \ -do { \ - UWtype __x0, __x1, __x2, __x3; \ - UHWtype __ul, __vl, __uh, __vh; \ - \ - __ul = __ll_lowpart(u); \ - __uh = __ll_highpart(u); \ - __vl = __ll_lowpart(v); \ - __vh = __ll_highpart(v); \ - \ - __x0 = (UWtype) __ul * __vl; \ - __x1 = (UWtype) __ul * __vh; \ - __x2 = (UWtype) __uh * __vl; \ - __x3 = (UWtype) __uh * __vh; \ - \ - __x1 += __ll_highpart(__x0); \ - __x1 += __x2; \ - if (__x1 < __x2) \ - __x3 += __ll_B; \ - \ - (w1) = __x3 + __ll_highpart(__x1); \ - (w0) = __ll_lowpart(__x1) * __ll_B + __ll_lowpart(__x0); \ -} while (0) - -#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ -do { \ - UWtype __x; \ - __x = (al) + (bl); \ - (sh) = (ah) + (bh) + (__x < (al)); \ - (sl) = __x; \ -} while (0) - -#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ -do { \ - UWtype __x; \ - __x = (al) - (bl); \ - (sh) = (ah) - (bh) - (__x > (al)); \ - (sl) = __x; \ -} while (0) - -#define udiv_qrnnd(q, r, n1, n0, d) \ -do { \ - UWtype __d1, __d0, __q1, __q0, __r1, __r0, __m; \ - __d1 = __ll_highpart(d); \ - __d0 = __ll_lowpart(d); \ - \ - __r1 = (n1) % __d1; \ - __q1 = (n1) / __d1; \ - __m = (UWtype) __q1 * __d0; \ - __r1 = __r1 * __ll_B | __ll_highpart(n0); \ - if (__r1 < __m) { \ - __q1--, __r1 += (d); \ - if (__r1 >= (d)) \ - if (__r1 < __m) \ - __q1--, __r1 += (d); \ - } \ - __r1 -= __m; \ - __r0 = __r1 % __d1; \ - __q0 = __r1 / __d1; \ - __m = (UWtype) __q0 * __d0; \ - __r0 = __r0 * __ll_B | __ll_lowpart(n0); \ - if (__r0 < __m) { \ - __q0--, __r0 += (d); \ - if (__r0 >= (d)) \ - if (__r0 < __m) \ - __q0--, __r0 += (d); \ - } \ - __r0 -= __m; \ - (q) = (UWtype) __q1 * __ll_B | __q0; \ - (r) = __r0; \ -} while (0) diff --git a/arch/nds32/include/asm/shmparam.h b/arch/nds32/include/asm/shmparam.h deleted file mode 100644 index 3aeee946973d..000000000000 --- a/arch/nds32/include/asm/shmparam.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMNDS32_SHMPARAM_H -#define _ASMNDS32_SHMPARAM_H - -/* - * This should be the size of the virtually indexed cache/ways, - * whichever is greater since the cache aliases every size/ways - * bytes. - */ -#define SHMLBA (4 * SZ_8K) /* attach addr a multiple of this */ - -/* - * Enforce SHMLBA in shmat - */ -#define __ARCH_FORCE_SHMLBA - -#endif /* _ASMNDS32_SHMPARAM_H */ diff --git a/arch/nds32/include/asm/stacktrace.h b/arch/nds32/include/asm/stacktrace.h deleted file mode 100644 index 6bf7c777bda4..000000000000 --- a/arch/nds32/include/asm/stacktrace.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2008-2018 Andes Technology Corporation */ - -#ifndef __ASM_STACKTRACE_H -#define __ASM_STACKTRACE_H - -/* Kernel callchain */ -struct stackframe { - unsigned long fp; - unsigned long sp; - unsigned long lp; -}; - -/* - * struct frame_tail: User callchain - * IMPORTANT: - * This struct is used for call-stack walking, - * the order and types matters. - * Do not use array, it only stores sizeof(pointer) - * - * The details can refer to arch/arm/kernel/perf_event.c - */ -struct frame_tail { - unsigned long stack_fp; - unsigned long stack_lp; -}; - -/* For User callchain with optimize for size */ -struct frame_tail_opt_size { - unsigned long stack_r6; - unsigned long stack_fp; - unsigned long stack_gp; - unsigned long stack_lp; -}; - -extern void -get_real_ret_addr(unsigned long *addr, struct task_struct *tsk, int *graph); - -#endif /* __ASM_STACKTRACE_H */ diff --git a/arch/nds32/include/asm/string.h b/arch/nds32/include/asm/string.h deleted file mode 100644 index cae8fe16de98..000000000000 --- a/arch/nds32/include/asm/string.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_STRING_H -#define __ASM_NDS32_STRING_H - -#define __HAVE_ARCH_MEMCPY -extern void *memcpy(void *, const void *, __kernel_size_t); - -#define __HAVE_ARCH_MEMMOVE -extern void *memmove(void *, const void *, __kernel_size_t); - -#define __HAVE_ARCH_MEMSET -extern void *memset(void *, int, __kernel_size_t); - -extern void *memzero(void *ptr, __kernel_size_t n); -#endif diff --git a/arch/nds32/include/asm/suspend.h b/arch/nds32/include/asm/suspend.h deleted file mode 100644 index 6ed2418af1ac..000000000000 --- a/arch/nds32/include/asm/suspend.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2008-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_SUSPEND_H -#define __ASM_NDS32_SUSPEND_H - -extern void suspend2ram(void); -extern void cpu_resume(void); -extern unsigned long wake_mask; - -#endif diff --git a/arch/nds32/include/asm/swab.h b/arch/nds32/include/asm/swab.h deleted file mode 100644 index 362a466f2976..000000000000 --- a/arch/nds32/include/asm/swab.h +++ /dev/null @@ -1,35 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_SWAB_H__ -#define __NDS32_SWAB_H__ - -#include <linux/types.h> -#include <linux/compiler.h> - -static __inline__ __attribute_const__ __u32 ___arch__swab32(__u32 x) -{ - __asm__("wsbh %0, %0\n\t" /* word swap byte within halfword */ - "rotri %0, %0, #16\n" - :"=r"(x) - :"0"(x)); - return x; -} - -static __inline__ __attribute_const__ __u16 ___arch__swab16(__u16 x) -{ - __asm__("wsbh %0, %0\n" /* word swap byte within halfword */ - :"=r"(x) - :"0"(x)); - return x; -} - -#define __arch_swab32(x) ___arch__swab32(x) -#define __arch_swab16(x) ___arch__swab16(x) - -#if !defined(__STRICT_ANSI__) || defined(__KERNEL__) -#define __BYTEORDER_HAS_U64__ -#define __SWAB_64_THRU_32__ -#endif - -#endif /* __NDS32_SWAB_H__ */ diff --git a/arch/nds32/include/asm/syscall.h b/arch/nds32/include/asm/syscall.h deleted file mode 100644 index 90aa56c94af1..000000000000 --- a/arch/nds32/include/asm/syscall.h +++ /dev/null @@ -1,142 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2008-2009 Red Hat, Inc. All rights reserved. -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASM_NDS32_SYSCALL_H -#define _ASM_NDS32_SYSCALL_H 1 - -#include <uapi/linux/audit.h> -#include <linux/err.h> -struct task_struct; -struct pt_regs; - -/** - * syscall_get_nr - find what system call a task is executing - * @task: task of interest, must be blocked - * @regs: task_pt_regs() of @task - * - * If @task is executing a system call or is at system call - * tracing about to attempt one, returns the system call number. - * If @task is not executing a system call, i.e. it's blocked - * inside the kernel for a fault or signal, returns -1. - * - * Note this returns int even on 64-bit machines. Only 32 bits of - * system call number can be meaningful. If the actual arch value - * is 64 bits, this truncates to 32 bits so 0xffffffff means -1. - * - * It's only valid to call this when @task is known to be blocked. - */ -static inline int -syscall_get_nr(struct task_struct *task, struct pt_regs *regs) -{ - return regs->syscallno; -} - -/** - * syscall_rollback - roll back registers after an aborted system call - * @task: task of interest, must be in system call exit tracing - * @regs: task_pt_regs() of @task - * - * It's only valid to call this when @task is stopped for system - * call exit tracing (due to TIF_SYSCALL_TRACE or TIF_SYSCALL_AUDIT), - * after tracehook_report_syscall_entry() returned nonzero to prevent - * the system call from taking place. - * - * This rolls back the register state in @regs so it's as if the - * system call instruction was a no-op. The registers containing - * the system call number and arguments are as they were before the - * system call instruction. This may not be the same as what the - * register state looked like at system call entry tracing. - */ -static inline void -syscall_rollback(struct task_struct *task, struct pt_regs *regs) -{ - regs->uregs[0] = regs->orig_r0; -} - -/** - * syscall_get_error - check result of traced system call - * @task: task of interest, must be blocked - * @regs: task_pt_regs() of @task - * - * Returns 0 if the system call succeeded, or -ERRORCODE if it failed. - * - * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - */ -static inline long -syscall_get_error(struct task_struct *task, struct pt_regs *regs) -{ - unsigned long error = regs->uregs[0]; - return IS_ERR_VALUE(error) ? error : 0; -} - -/** - * syscall_get_return_value - get the return value of a traced system call - * @task: task of interest, must be blocked - * @regs: task_pt_regs() of @task - * - * Returns the return value of the successful system call. - * This value is meaningless if syscall_get_error() returned nonzero. - * - * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - */ -static inline long -syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) -{ - return regs->uregs[0]; -} - -/** - * syscall_set_return_value - change the return value of a traced system call - * @task: task of interest, must be blocked - * @regs: task_pt_regs() of @task - * @error: negative error code, or zero to indicate success - * @val: user return value if @error is zero - * - * This changes the results of the system call that user mode will see. - * If @error is zero, the user sees a successful system call with a - * return value of @val. If @error is nonzero, it's a negated errno - * code; the user sees a failed system call with this errno code. - * - * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - */ -static inline void -syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, - int error, long val) -{ - regs->uregs[0] = (long)error ? error : val; -} - -/** - * syscall_get_arguments - extract system call parameter values - * @task: task of interest, must be blocked - * @regs: task_pt_regs() of @task - * @args: array filled with argument values - * - * Fetches 6 arguments to the system call (from 0 through 5). The first - * argument is stored in @args[0], and so on. - * - * It's only valid to call this when @task is stopped for tracing on - * entry to a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - */ -#define SYSCALL_MAX_ARGS 6 -static inline void -syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned long *args) -{ - args[0] = regs->orig_r0; - args++; - memcpy(args, ®s->uregs[0] + 1, 5 * sizeof(args[0])); -} - -static inline int -syscall_get_arch(struct task_struct *task) -{ - return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) - ? AUDIT_ARCH_NDS32BE : AUDIT_ARCH_NDS32; -} - -#endif /* _ASM_NDS32_SYSCALL_H */ diff --git a/arch/nds32/include/asm/syscalls.h b/arch/nds32/include/asm/syscalls.h deleted file mode 100644 index 4e7216082a67..000000000000 --- a/arch/nds32/include/asm/syscalls.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_SYSCALLS_H -#define __ASM_NDS32_SYSCALLS_H - -asmlinkage long sys_cacheflush(unsigned long addr, unsigned long len, unsigned int op); -asmlinkage long sys_fadvise64_64_wrapper(int fd, int advice, loff_t offset, loff_t len); -asmlinkage long sys_rt_sigreturn_wrapper(void); -asmlinkage long sys_fp_udfiex_crtl(int cmd, int act); - -#include <asm-generic/syscalls.h> - -#endif /* __ASM_NDS32_SYSCALLS_H */ diff --git a/arch/nds32/include/asm/thread_info.h b/arch/nds32/include/asm/thread_info.h deleted file mode 100644 index d3967ad184f0..000000000000 --- a/arch/nds32/include/asm/thread_info.h +++ /dev/null @@ -1,76 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_THREAD_INFO_H -#define __ASM_NDS32_THREAD_INFO_H - -#ifdef __KERNEL__ - -#define THREAD_SIZE_ORDER (1) -#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) - -#ifndef __ASSEMBLY__ - -struct task_struct; - -#include <asm/ptrace.h> -#include <asm/types.h> - -typedef unsigned long mm_segment_t; - -/* - * low level task data that entry.S needs immediate access to. - * __switch_to() assumes cpu_context follows immediately after cpu_domain. - */ -struct thread_info { - unsigned long flags; /* low level flags */ - __s32 preempt_count; /* 0 => preemptable, <0 => bug */ - mm_segment_t addr_limit; /* address limit */ -}; -#define INIT_THREAD_INFO(tsk) \ -{ \ - .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ -} -#define thread_saved_pc(tsk) ((unsigned long)(tsk->thread.cpu_context.pc)) -#define thread_saved_fp(tsk) ((unsigned long)(tsk->thread.cpu_context.fp)) -#endif - -/* - * thread information flags: - * TIF_SYSCALL_TRACE - syscall trace active - * TIF_SIGPENDING - signal pending - * TIF_NEED_RESCHED - rescheduling necessary - * TIF_NOTIFY_RESUME - callback before returning to user - * TIF_POLLING_NRFLAG - true if poll_idle() is polling TIF_NEED_RESCHED - */ -#define TIF_SIGPENDING 1 -#define TIF_NEED_RESCHED 2 -#define TIF_SINGLESTEP 3 -#define TIF_NOTIFY_RESUME 4 /* callback before returning to user */ -#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ -#define TIF_SYSCALL_TRACE 8 -#define TIF_POLLING_NRFLAG 17 -#define TIF_MEMDIE 18 -#define TIF_FREEZE 19 -#define TIF_RESTORE_SIGMASK 20 - -#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) -#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) -#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) -#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) -#define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) -#define _TIF_FREEZE (1 << TIF_FREEZE) -#define _TIF_RESTORE_SIGMASK (1 << TIF_RESTORE_SIGMASK) - -/* - * Change these and you break ASM code in entry-common.S - */ -#define _TIF_WORK_MASK 0x000000ff -#define _TIF_WORK_SYSCALL_ENTRY (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP) -#define _TIF_WORK_SYSCALL_LEAVE (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP) - -#endif /* __KERNEL__ */ -#endif /* __ASM_NDS32_THREAD_INFO_H */ diff --git a/arch/nds32/include/asm/tlb.h b/arch/nds32/include/asm/tlb.h deleted file mode 100644 index 672603804a3b..000000000000 --- a/arch/nds32/include/asm/tlb.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASMNDS32_TLB_H -#define __ASMNDS32_TLB_H - -#include <asm-generic/tlb.h> - -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) - -#endif diff --git a/arch/nds32/include/asm/tlbflush.h b/arch/nds32/include/asm/tlbflush.h deleted file mode 100644 index 97155366ea01..000000000000 --- a/arch/nds32/include/asm/tlbflush.h +++ /dev/null @@ -1,46 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMNDS32_TLBFLUSH_H -#define _ASMNDS32_TLBFLUSH_H - -#include <linux/spinlock.h> -#include <linux/mm.h> -#include <nds32_intrinsic.h> - -static inline void local_flush_tlb_all(void) -{ - __nds32__tlbop_flua(); - __nds32__isb(); -} - -static inline void local_flush_tlb_mm(struct mm_struct *mm) -{ - __nds32__tlbop_flua(); - __nds32__isb(); -} - -static inline void local_flush_tlb_kernel_range(unsigned long start, - unsigned long end) -{ - while (start < end) { - __nds32__tlbop_inv(start); - __nds32__isb(); - start += PAGE_SIZE; - } -} - -void local_flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end); -void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); - -#define flush_tlb_all local_flush_tlb_all -#define flush_tlb_mm local_flush_tlb_mm -#define flush_tlb_range local_flush_tlb_range -#define flush_tlb_page local_flush_tlb_page -#define flush_tlb_kernel_range local_flush_tlb_kernel_range - -void update_mmu_cache(struct vm_area_struct *vma, - unsigned long address, pte_t * pte); - -#endif diff --git a/arch/nds32/include/asm/uaccess.h b/arch/nds32/include/asm/uaccess.h deleted file mode 100644 index d4cbf069dc22..000000000000 --- a/arch/nds32/include/asm/uaccess.h +++ /dev/null @@ -1,286 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMANDES_UACCESS_H -#define _ASMANDES_UACCESS_H - -/* - * User space memory access functions - */ -#include <linux/sched.h> -#include <asm/errno.h> -#include <asm/memory.h> -#include <asm/types.h> - -#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); - -#define KERNEL_DS ((mm_segment_t) { ~0UL }) -#define USER_DS ((mm_segment_t) {TASK_SIZE - 1}) - -#define get_fs() (current_thread_info()->addr_limit) -#define user_addr_max get_fs - -static inline void set_fs(mm_segment_t fs) -{ - current_thread_info()->addr_limit = fs; -} - -#define uaccess_kernel() (get_fs() == KERNEL_DS) - -#define __range_ok(addr, size) (size <= get_fs() && addr <= (get_fs() -size)) - -#define access_ok(addr, size) \ - __range_ok((unsigned long)addr, (unsigned long)size) -/* - * Single-value transfer routines. They automatically use the right - * size if we just have the right pointer type. Note that the functions - * which read from user space (*get_*) need to take care not to leak - * kernel data even if the calling code is buggy and fails to check - * the return value. This means zeroing out the destination variable - * or buffer on error. Normally this is done out of line by the - * fixup code, but there are a few places where it intrudes on the - * main code path. When we only write to user space, there is no - * problem. - * - * The "__xxx" versions of the user access functions do not verify the - * address space - it must have been done previously with a separate - * "access_ok()" call. - * - * The "xxx_error" versions set the third argument to EFAULT if an - * error occurs, and leave it unchanged on success. Note that these - * versions are void (ie, don't return a value as such). - */ - -#define get_user __get_user \ - -#define __get_user(x, ptr) \ -({ \ - long __gu_err = 0; \ - __get_user_check((x), (ptr), __gu_err); \ - __gu_err; \ -}) - -#define __get_user_error(x, ptr, err) \ -({ \ - __get_user_check((x), (ptr), (err)); \ - (void)0; \ -}) - -#define __get_user_check(x, ptr, err) \ -({ \ - const __typeof__(*(ptr)) __user *__p = (ptr); \ - might_fault(); \ - if (access_ok(__p, sizeof(*__p))) { \ - __get_user_err((x), __p, (err)); \ - } else { \ - (x) = 0; (err) = -EFAULT; \ - } \ -}) - -#define __get_user_err(x, ptr, err) \ -do { \ - unsigned long __gu_val; \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __get_user_asm("lbi", __gu_val, (ptr), (err)); \ - break; \ - case 2: \ - __get_user_asm("lhi", __gu_val, (ptr), (err)); \ - break; \ - case 4: \ - __get_user_asm("lwi", __gu_val, (ptr), (err)); \ - break; \ - case 8: \ - __get_user_asm_dword(__gu_val, (ptr), (err)); \ - break; \ - default: \ - BUILD_BUG(); \ - break; \ - } \ - (x) = (__force __typeof__(*(ptr)))__gu_val; \ -} while (0) - -#define __get_user_asm(inst, x, addr, err) \ - __asm__ __volatile__ ( \ - "1: "inst" %1,[%2]\n" \ - "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: move %0, %3\n" \ - " move %1, #0\n" \ - " b 2b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .previous" \ - : "+r" (err), "=&r" (x) \ - : "r" (addr), "i" (-EFAULT) \ - : "cc") - -#ifdef __NDS32_EB__ -#define __gu_reg_oper0 "%H1" -#define __gu_reg_oper1 "%L1" -#else -#define __gu_reg_oper0 "%L1" -#define __gu_reg_oper1 "%H1" -#endif - -#define __get_user_asm_dword(x, addr, err) \ - __asm__ __volatile__ ( \ - "\n1:\tlwi " __gu_reg_oper0 ",[%2]\n" \ - "\n2:\tlwi " __gu_reg_oper1 ",[%2+4]\n" \ - "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .align 2\n" \ - "4: move %0, %3\n" \ - " b 3b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 4b\n" \ - " .long 2b, 4b\n" \ - " .previous" \ - : "+r"(err), "=&r"(x) \ - : "r"(addr), "i"(-EFAULT) \ - : "cc") - -#define put_user __put_user \ - -#define __put_user(x, ptr) \ -({ \ - long __pu_err = 0; \ - __put_user_err((x), (ptr), __pu_err); \ - __pu_err; \ -}) - -#define __put_user_error(x, ptr, err) \ -({ \ - __put_user_err((x), (ptr), (err)); \ - (void)0; \ -}) - -#define __put_user_check(x, ptr, err) \ -({ \ - __typeof__(*(ptr)) __user *__p = (ptr); \ - might_fault(); \ - if (access_ok(__p, sizeof(*__p))) { \ - __put_user_err((x), __p, (err)); \ - } else { \ - (err) = -EFAULT; \ - } \ -}) - -#define __put_user_err(x, ptr, err) \ -do { \ - __typeof__(*(ptr)) __pu_val = (x); \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __put_user_asm("sbi", __pu_val, (ptr), (err)); \ - break; \ - case 2: \ - __put_user_asm("shi", __pu_val, (ptr), (err)); \ - break; \ - case 4: \ - __put_user_asm("swi", __pu_val, (ptr), (err)); \ - break; \ - case 8: \ - __put_user_asm_dword(__pu_val, (ptr), (err)); \ - break; \ - default: \ - BUILD_BUG(); \ - break; \ - } \ -} while (0) - -#define __put_user_asm(inst, x, addr, err) \ - __asm__ __volatile__ ( \ - "1: "inst" %1,[%2]\n" \ - "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: move %0, %3\n" \ - " b 2b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .previous" \ - : "+r" (err) \ - : "r" (x), "r" (addr), "i" (-EFAULT) \ - : "cc") - -#ifdef __NDS32_EB__ -#define __pu_reg_oper0 "%H2" -#define __pu_reg_oper1 "%L2" -#else -#define __pu_reg_oper0 "%L2" -#define __pu_reg_oper1 "%H2" -#endif - -#define __put_user_asm_dword(x, addr, err) \ - __asm__ __volatile__ ( \ - "\n1:\tswi " __pu_reg_oper0 ",[%1]\n" \ - "\n2:\tswi " __pu_reg_oper1 ",[%1+4]\n" \ - "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .align 2\n" \ - "4: move %0, %3\n" \ - " b 3b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 4b\n" \ - " .long 2b, 4b\n" \ - " .previous" \ - : "+r"(err) \ - : "r"(addr), "r"(x), "i"(-EFAULT) \ - : "cc") - -extern unsigned long __arch_clear_user(void __user * addr, unsigned long n); -extern long strncpy_from_user(char *dest, const char __user * src, long count); -extern __must_check long strnlen_user(const char __user * str, long n); -extern unsigned long __arch_copy_from_user(void *to, const void __user * from, - unsigned long n); -extern unsigned long __arch_copy_to_user(void __user * to, const void *from, - unsigned long n); - -#define raw_copy_from_user __arch_copy_from_user -#define raw_copy_to_user __arch_copy_to_user - -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER -static inline unsigned long clear_user(void __user * to, unsigned long n) -{ - if (access_ok(to, n)) - n = __arch_clear_user(to, n); - return n; -} - -static inline unsigned long __clear_user(void __user * to, unsigned long n) -{ - return __arch_clear_user(to, n); -} - -#endif /* _ASMNDS32_UACCESS_H */ diff --git a/arch/nds32/include/asm/unistd.h b/arch/nds32/include/asm/unistd.h deleted file mode 100644 index bf5e2d440913..000000000000 --- a/arch/nds32/include/asm/unistd.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#define __ARCH_WANT_SYS_CLONE - -#include <uapi/asm/unistd.h> diff --git a/arch/nds32/include/asm/vdso.h b/arch/nds32/include/asm/vdso.h deleted file mode 100644 index 89b113ffc3dc..000000000000 --- a/arch/nds32/include/asm/vdso.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2005-2017 Andes Technology Corporation - */ - -#ifndef __ASM_VDSO_H -#define __ASM_VDSO_H - -#ifdef __KERNEL__ - -#ifndef __ASSEMBLY__ - -#include <generated/vdso-offsets.h> - -#define VDSO_SYMBOL(base, name) \ -({ \ - (unsigned long)(vdso_offset_##name + (unsigned long)(base)); \ -}) - -#endif /* !__ASSEMBLY__ */ - -#endif /* __KERNEL__ */ - -#endif /* __ASM_VDSO_H */ diff --git a/arch/nds32/include/asm/vdso_datapage.h b/arch/nds32/include/asm/vdso_datapage.h deleted file mode 100644 index 74c68802021e..000000000000 --- a/arch/nds32/include/asm/vdso_datapage.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2012 ARM Limited -// Copyright (C) 2005-2017 Andes Technology Corporation -#ifndef __ASM_VDSO_DATAPAGE_H -#define __ASM_VDSO_DATAPAGE_H - -#ifdef __KERNEL__ - -#ifndef __ASSEMBLY__ - -struct vdso_data { - bool cycle_count_down; /* timer cyclye counter is decrease with time */ - u32 cycle_count_offset; /* offset of timer cycle counter register */ - u32 seq_count; /* sequence count - odd during updates */ - u32 xtime_coarse_sec; /* coarse time */ - u32 xtime_coarse_nsec; - - u32 wtm_clock_sec; /* wall to monotonic offset */ - u32 wtm_clock_nsec; - u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */ - u32 cs_mult; /* clocksource multiplier */ - u32 cs_shift; /* Cycle to nanosecond divisor (power of two) */ - u32 hrtimer_res; /* hrtimer resolution */ - - u64 cs_cycle_last; /* last cycle value */ - u64 cs_mask; /* clocksource mask */ - - u64 xtime_clock_nsec; /* CLOCK_REALTIME sub-ns base */ - u32 tz_minuteswest; /* timezone info for gettimeofday(2) */ - u32 tz_dsttime; -}; - -#endif /* !__ASSEMBLY__ */ - -#endif /* __KERNEL__ */ - -#endif /* __ASM_VDSO_DATAPAGE_H */ diff --git a/arch/nds32/include/asm/vdso_timer_info.h b/arch/nds32/include/asm/vdso_timer_info.h deleted file mode 100644 index 328439ce37db..000000000000 --- a/arch/nds32/include/asm/vdso_timer_info.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -extern struct timer_info_t timer_info; -#define EMPTY_VALUE ~(0UL) -#define EMPTY_TIMER_MAPPING EMPTY_VALUE -#define EMPTY_REG_OFFSET EMPTY_VALUE - -struct timer_info_t -{ - bool cycle_count_down; - unsigned long mapping_base; - unsigned long cycle_count_reg_offset; -}; diff --git a/arch/nds32/include/asm/vermagic.h b/arch/nds32/include/asm/vermagic.h deleted file mode 100644 index f772e7ba33f1..000000000000 --- a/arch/nds32/include/asm/vermagic.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASM_VERMAGIC_H -#define _ASM_VERMAGIC_H - -#define MODULE_ARCH_VERMAGIC "NDS32v3" - -#endif /* _ASM_VERMAGIC_H */ diff --git a/arch/nds32/include/asm/vmalloc.h b/arch/nds32/include/asm/vmalloc.h deleted file mode 100644 index caeed3898419..000000000000 --- a/arch/nds32/include/asm/vmalloc.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef _ASM_NDS32_VMALLOC_H -#define _ASM_NDS32_VMALLOC_H - -#endif /* _ASM_NDS32_VMALLOC_H */ diff --git a/arch/nds32/include/uapi/asm/Kbuild b/arch/nds32/include/uapi/asm/Kbuild deleted file mode 100644 index e78470141932..000000000000 --- a/arch/nds32/include/uapi/asm/Kbuild +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -generic-y += ucontext.h diff --git a/arch/nds32/include/uapi/asm/auxvec.h b/arch/nds32/include/uapi/asm/auxvec.h deleted file mode 100644 index bc0b92ab8c15..000000000000 --- a/arch/nds32/include/uapi/asm/auxvec.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_AUXVEC_H -#define __ASM_AUXVEC_H - -/* - * This entry gives some information about the FPU initialization - * performed by the kernel. - */ -#define AT_FPUCW 18 /* Used FPU control word. */ - - -/* VDSO location */ -#define AT_SYSINFO_EHDR 33 - -#define AT_VECTOR_SIZE_ARCH 1 - -#endif diff --git a/arch/nds32/include/uapi/asm/byteorder.h b/arch/nds32/include/uapi/asm/byteorder.h deleted file mode 100644 index c264ef12c49c..000000000000 --- a/arch/nds32/include/uapi/asm/byteorder.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __NDS32_BYTEORDER_H__ -#define __NDS32_BYTEORDER_H__ - -#ifdef __NDS32_EB__ -#include <linux/byteorder/big_endian.h> -#else -#include <linux/byteorder/little_endian.h> -#endif - -#endif /* __NDS32_BYTEORDER_H__ */ diff --git a/arch/nds32/include/uapi/asm/cachectl.h b/arch/nds32/include/uapi/asm/cachectl.h deleted file mode 100644 index 31b9b439d819..000000000000 --- a/arch/nds32/include/uapi/asm/cachectl.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 1994, 1995, 1996 by Ralf Baechle -// Copyright (C) 2005-2017 Andes Technology Corporation -#ifndef _ASM_CACHECTL -#define _ASM_CACHECTL - -/* - * Options for cacheflush system call - */ -#define ICACHE 0 /* flush instruction cache */ -#define DCACHE 1 /* writeback and flush data cache */ -#define BCACHE 2 /* flush instruction cache + writeback and flush data cache */ - -#endif /* _ASM_CACHECTL */ diff --git a/arch/nds32/include/uapi/asm/fp_udfiex_crtl.h b/arch/nds32/include/uapi/asm/fp_udfiex_crtl.h deleted file mode 100644 index f17396db16ec..000000000000 --- a/arch/nds32/include/uapi/asm/fp_udfiex_crtl.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* Copyright (C) 2005-2019 Andes Technology Corporation */ -#ifndef _FP_UDF_IEX_CRTL_H -#define _FP_UDF_IEX_CRTL_H - -/* - * The cmd list of sys_fp_udfiex_crtl() - */ -/* Disable UDF or IEX trap based on the content of parameter act */ -#define DISABLE_UDF_IEX_TRAP 0 -/* Enable UDF or IEX trap based on the content of parameter act */ -#define ENABLE_UDF_IEX_TRAP 1 -/* Get current status of UDF and IEX trap */ -#define GET_UDF_IEX_TRAP 2 - -#endif /* _FP_UDF_IEX_CRTL_H */ diff --git a/arch/nds32/include/uapi/asm/param.h b/arch/nds32/include/uapi/asm/param.h deleted file mode 100644 index 48d00328d328..000000000000 --- a/arch/nds32/include/uapi/asm/param.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __ASM_NDS32_PARAM_H -#define __ASM_NDS32_PARAM_H - -#define EXEC_PAGESIZE 8192 - -#include <asm-generic/param.h> - -#endif /* __ASM_NDS32_PARAM_H */ diff --git a/arch/nds32/include/uapi/asm/ptrace.h b/arch/nds32/include/uapi/asm/ptrace.h deleted file mode 100644 index d76217c7c010..000000000000 --- a/arch/nds32/include/uapi/asm/ptrace.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef __UAPI_ASM_NDS32_PTRACE_H -#define __UAPI_ASM_NDS32_PTRACE_H - -#ifndef __ASSEMBLY__ - -/* - * User structures for general purpose register. - */ -struct user_pt_regs { - long uregs[26]; - long fp; - long gp; - long lp; - long sp; - long ipc; - long lb; - long le; - long lc; - long syscallno; -}; -#endif -#endif diff --git a/arch/nds32/include/uapi/asm/sigcontext.h b/arch/nds32/include/uapi/asm/sigcontext.h deleted file mode 100644 index 6c1e6648878f..000000000000 --- a/arch/nds32/include/uapi/asm/sigcontext.h +++ /dev/null @@ -1,84 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#ifndef _ASMNDS32_SIGCONTEXT_H -#define _ASMNDS32_SIGCONTEXT_H - -/* - * Signal context structure - contains all info to do with the state - * before the signal handler was invoked. Note: only add new entries - * to the end of the structure. - */ -struct fpu_struct { - unsigned long long fd_regs[32]; - unsigned long fpcsr; - /* - * When CONFIG_SUPPORT_DENORMAL_ARITHMETIC is defined, kernel prevents - * hardware from treating the denormalized output as an underflow case - * and rounding it to a normal number. Hence kernel enables the UDF and - * IEX trap in the fpcsr register to step in the calculation. - * However, the UDF and IEX trap enable bit in $fpcsr also lose - * their use. - * - * UDF_IEX_trap replaces the feature of UDF and IEX trap enable bit in - * $fpcsr to control the trap of underflow and inexact. The bit filed - * of UDF_IEX_trap is the same as $fpcsr, 10th bit is used to enable UDF - * exception trapping and 11th bit is used to enable IEX exception - * trapping. - * - * UDF_IEX_trap is only modified through fp_udfiex_crtl syscall. - * Therefore, UDF_IEX_trap needn't be saved and restored in each - * context switch. - */ - unsigned long UDF_IEX_trap; -}; - -struct zol_struct { - unsigned long nds32_lc; /* $LC */ - unsigned long nds32_le; /* $LE */ - unsigned long nds32_lb; /* $LB */ -}; - -struct sigcontext { - unsigned long trap_no; - unsigned long error_code; - unsigned long oldmask; - unsigned long nds32_r0; - unsigned long nds32_r1; - unsigned long nds32_r2; - unsigned long nds32_r3; - unsigned long nds32_r4; - unsigned long nds32_r5; - unsigned long nds32_r6; - unsigned long nds32_r7; - unsigned long nds32_r8; - unsigned long nds32_r9; - unsigned long nds32_r10; - unsigned long nds32_r11; - unsigned long nds32_r12; - unsigned long nds32_r13; - unsigned long nds32_r14; - unsigned long nds32_r15; - unsigned long nds32_r16; - unsigned long nds32_r17; - unsigned long nds32_r18; - unsigned long nds32_r19; - unsigned long nds32_r20; - unsigned long nds32_r21; - unsigned long nds32_r22; - unsigned long nds32_r23; - unsigned long nds32_r24; - unsigned long nds32_r25; - unsigned long nds32_fp; /* $r28 */ - unsigned long nds32_gp; /* $r29 */ - unsigned long nds32_lp; /* $r30 */ - unsigned long nds32_sp; /* $r31 */ - unsigned long nds32_ipc; - unsigned long fault_address; - unsigned long used_math_flag; - /* FPU Registers */ - struct fpu_struct fpu; - struct zol_struct zol; -}; - -#endif diff --git a/arch/nds32/include/uapi/asm/unistd.h b/arch/nds32/include/uapi/asm/unistd.h deleted file mode 100644 index 410795e280fe..000000000000 --- a/arch/nds32/include/uapi/asm/unistd.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2005-2017 Andes Technology Corporation - -#define __ARCH_WANT_STAT64 -#define __ARCH_WANT_SYNC_FILE_RANGE2 -#define __ARCH_WANT_SET_GET_RLIMIT -#define __ARCH_WANT_TIME32_SYSCALLS - -/* Use the standard ABI for syscalls */ -#include <asm-generic/unistd.h> - -/* Additional NDS32 specific syscalls. */ -#define __NR_cacheflush (__NR_arch_specific_syscall) -#define __NR_fp_udfiex_crtl (__NR_arch_specific_syscall + 1) -__SYSCALL(__NR_cacheflush, sys_cacheflush) -__SYSCALL(__NR_fp_udfiex_crtl, sys_fp_udfiex_crtl) diff --git a/arch/nds32/kernel/.gitignore b/arch/nds32/kernel/.gitignore deleted file mode 100644 index bbb90f92d051..000000000000 --- a/arch/nds32/kernel/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -vmlinux.lds diff --git a/arch/nds32/kernel/Makefile b/arch/nds32/kernel/Makefile deleted file mode 100644 index 394df3f6442c..000000000000 --- a/arch/nds32/kernel/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the linux kernel. -# - -CPPFLAGS_vmlinux.lds := -DTEXTADDR=$(TEXTADDR) -AFLAGS_head.o := -DTEXTADDR=$(TEXTADDR) -# Object file lists. - -obj-y := ex-entry.o ex-exit.o ex-scall.o irq.o \ - process.o ptrace.o setup.o signal.o \ - sys_nds32.o time.o traps.o cacheinfo.o \ - dma.o syscall_table.o vdso.o - -obj-$(CONFIG_MODULES) += nds32_ksyms.o module.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o -obj-$(CONFIG_FPU) += fpu.o -obj-$(CONFIG_OF) += devtree.o -obj-$(CONFIG_CACHE_L2) += atl2c.o -obj-$(CONFIG_PERF_EVENTS) += perf_event_cpu.o -obj-$(CONFIG_PM) += pm.o sleep.o -extra-y := head.o vmlinux.lds - -CFLAGS_fpu.o += -mext-fpu-sp -mext-fpu-dp - - -obj-y += vdso/ - -obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o - -ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) -endif diff --git a/arch/nds32/kernel/asm-offsets.c b/arch/nds32/kernel/asm-offsets.c deleted file mode 100644 index 3541d5981de7..000000000000 --- a/arch/nds32/kernel/asm-offsets.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/sched.h> -#include <linux/sched/task_stack.h> -#include <linux/kbuild.h> -#include <asm/thread_info.h> -#include <asm/ptrace.h> - -int main(void) -{ - DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); - DEFINE(TSK_TI_PREEMPT, - offsetof(struct task_struct, thread_info.preempt_count)); - DEFINE(THREAD_CPU_CONTEXT, - offsetof(struct task_struct, thread.cpu_context)); - DEFINE(OSP_OFFSET, offsetof(struct pt_regs, osp)); - DEFINE(SP_OFFSET, offsetof(struct pt_regs, sp)); - DEFINE(FUCOP_CTL_OFFSET, offsetof(struct pt_regs, fucop_ctl)); - DEFINE(IPSW_OFFSET, offsetof(struct pt_regs, ipsw)); - DEFINE(SYSCALLNO_OFFSET, offsetof(struct pt_regs, syscallno)); - DEFINE(IPC_OFFSET, offsetof(struct pt_regs, ipc)); - DEFINE(R0_OFFSET, offsetof(struct pt_regs, uregs[0])); - DEFINE(R15_OFFSET, offsetof(struct pt_regs, uregs[15])); - DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); - DEFINE(CLOCK_COARSE_RES, LOW_RES_NSEC); - return 0; -} diff --git a/arch/nds32/kernel/atl2c.c b/arch/nds32/kernel/atl2c.c deleted file mode 100644 index 0c5386e72098..000000000000 --- a/arch/nds32/kernel/atl2c.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/compiler.h> -#include <linux/of_address.h> -#include <linux/of_fdt.h> -#include <linux/of_platform.h> -#include <asm/l2_cache.h> - -void __iomem *atl2c_base; -static const struct of_device_id atl2c_ids[] __initconst = { - {.compatible = "andestech,atl2c",}, - {} -}; - -static int __init atl2c_of_init(void) -{ - struct device_node *np; - struct resource res; - unsigned long tmp = 0; - unsigned long l2set, l2way, l2clsz; - - if (!(__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskL2C)) - return -ENODEV; - - np = of_find_matching_node(NULL, atl2c_ids); - if (!np) - return -ENODEV; - - if (of_address_to_resource(np, 0, &res)) - return -ENODEV; - - atl2c_base = ioremap(res.start, resource_size(&res)); - if (!atl2c_base) - return -ENOMEM; - - l2set = - 64 << ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2SET) >> - L2_CA_CONF_offL2SET); - l2way = - 1 + - ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2WAY) >> - L2_CA_CONF_offL2WAY); - l2clsz = - 4 << ((L2C_R_REG(L2_CA_CONF_OFF) & L2_CA_CONF_mskL2CLSZ) >> - L2_CA_CONF_offL2CLSZ); - pr_info("L2:%luKB/%luS/%luW/%luB\n", - l2set * l2way * l2clsz / 1024, l2set, l2way, l2clsz); - - tmp = L2C_R_REG(L2CC_PROT_OFF); - tmp &= ~L2CC_PROT_mskMRWEN; - L2C_W_REG(L2CC_PROT_OFF, tmp); - - tmp = L2C_R_REG(L2CC_SETUP_OFF); - tmp &= ~L2CC_SETUP_mskPART; - L2C_W_REG(L2CC_SETUP_OFF, tmp); - - tmp = L2C_R_REG(L2CC_CTRL_OFF); - tmp |= L2CC_CTRL_mskEN; - L2C_W_REG(L2CC_CTRL_OFF, tmp); - - return 0; -} - -subsys_initcall(atl2c_of_init); diff --git a/arch/nds32/kernel/cacheinfo.c b/arch/nds32/kernel/cacheinfo.c deleted file mode 100644 index aab98e447feb..000000000000 --- a/arch/nds32/kernel/cacheinfo.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/bitops.h> -#include <linux/cacheinfo.h> -#include <linux/cpu.h> - -static void ci_leaf_init(struct cacheinfo *this_leaf, - enum cache_type type, unsigned int level) -{ - char cache_type = (type & CACHE_TYPE_INST ? ICACHE : DCACHE); - - this_leaf->level = level; - this_leaf->type = type; - this_leaf->coherency_line_size = CACHE_LINE_SIZE(cache_type); - this_leaf->number_of_sets = CACHE_SET(cache_type); - this_leaf->ways_of_associativity = CACHE_WAY(cache_type); - this_leaf->size = this_leaf->number_of_sets * - this_leaf->coherency_line_size * this_leaf->ways_of_associativity; -#if defined(CONFIG_CPU_DCACHE_WRITETHROUGH) - this_leaf->attributes = CACHE_WRITE_THROUGH; -#else - this_leaf->attributes = CACHE_WRITE_BACK; -#endif -} - -int init_cache_level(unsigned int cpu) -{ - struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - - /* Only 1 level and I/D cache seperate. */ - this_cpu_ci->num_levels = 1; - this_cpu_ci->num_leaves = 2; - return 0; -} - -int populate_cache_leaves(unsigned int cpu) -{ - unsigned int level, idx; - struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - struct cacheinfo *this_leaf = this_cpu_ci->info_list; - - for (idx = 0, level = 1; level <= this_cpu_ci->num_levels && - idx < this_cpu_ci->num_leaves; idx++, level++) { - ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); - ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); - } - return 0; -} diff --git a/arch/nds32/kernel/devtree.c b/arch/nds32/kernel/devtree.c deleted file mode 100644 index bdce0fe5af9f..000000000000 --- a/arch/nds32/kernel/devtree.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/bug.h> -#include <linux/printk.h> -#include <linux/of_fdt.h> - -void __init early_init_devtree(void *params) -{ - if (!params || !early_init_dt_scan(params)) { - pr_crit("\n" - "Error: invalid device tree blob at (virtual address 0x%p)\n" - "\nPlease check your bootloader.", params); - - BUG_ON(1); - } - - dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); -} diff --git a/arch/nds32/kernel/dma.c b/arch/nds32/kernel/dma.c deleted file mode 100644 index 2ac8e6c82a61..000000000000 --- a/arch/nds32/kernel/dma.c +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/types.h> -#include <linux/mm.h> -#include <linux/dma-map-ops.h> -#include <linux/cache.h> -#include <linux/highmem.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> -#include <asm/proc-fns.h> - -static inline void cache_op(phys_addr_t paddr, size_t size, - void (*fn)(unsigned long start, unsigned long end)) -{ - struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); - unsigned offset = paddr & ~PAGE_MASK; - size_t left = size; - unsigned long start; - - do { - size_t len = left; - - if (PageHighMem(page)) { - void *addr; - - if (offset + len > PAGE_SIZE) { - if (offset >= PAGE_SIZE) { - page += offset >> PAGE_SHIFT; - offset &= ~PAGE_MASK; - } - len = PAGE_SIZE - offset; - } - - addr = kmap_atomic(page); - start = (unsigned long)(addr + offset); - fn(start, start + len); - kunmap_atomic(addr); - } else { - start = (unsigned long)phys_to_virt(paddr); - fn(start, start + size); - } - offset = 0; - page++; - left -= len; - } while (left); -} - -void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - switch (dir) { - case DMA_FROM_DEVICE: - break; - case DMA_TO_DEVICE: - case DMA_BIDIRECTIONAL: - cache_op(paddr, size, cpu_dma_wb_range); - break; - default: - BUG(); - } -} - -void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - switch (dir) { - case DMA_TO_DEVICE: - break; - case DMA_FROM_DEVICE: - case DMA_BIDIRECTIONAL: - cache_op(paddr, size, cpu_dma_inval_range); - break; - default: - BUG(); - } -} - -void arch_dma_prep_coherent(struct page *page, size_t size) -{ - cache_op(page_to_phys(page), size, cpu_dma_wbinval_range); -} diff --git a/arch/nds32/kernel/ex-entry.S b/arch/nds32/kernel/ex-entry.S deleted file mode 100644 index 107d98a1d1b8..000000000000 --- a/arch/nds32/kernel/ex-entry.S +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/memory.h> -#include <asm/nds32.h> -#include <asm/errno.h> -#include <asm/asm-offsets.h> -#include <asm/page.h> -#include <asm/fpu.h> - -#ifdef CONFIG_HWZOL - .macro push_zol - mfusr $r14, $LB - mfusr $r15, $LE - mfusr $r16, $LC - .endm -#endif - .macro skip_save_fucop_ctl -#if defined(CONFIG_FPU) -skip_fucop_ctl: - smw.adm $p0, [$sp], $p0, #0x1 - j fucop_ctl_done -#endif - .endm - - .macro save_user_regs -#if defined(CONFIG_FPU) - sethi $p0, hi20(has_fpu) - lbsi $p0, [$p0+lo12(has_fpu)] - beqz $p0, skip_fucop_ctl - mfsr $p0, $FUCOP_CTL - smw.adm $p0, [$sp], $p0, #0x1 - bclr $p0, $p0, #FUCOP_CTL_offCP0EN - mtsr $p0, $FUCOP_CTL -fucop_ctl_done: - /* move $SP to the bottom of pt_regs */ - addi $sp, $sp, -FUCOP_CTL_OFFSET -#else - smw.adm $sp, [$sp], $sp, #0x1 - /* move $SP to the bottom of pt_regs */ - addi $sp, $sp, -OSP_OFFSET -#endif - - /* push $r0 ~ $r25 */ - smw.bim $r0, [$sp], $r25 - /* push $fp, $gp, $lp */ - smw.bim $sp, [$sp], $sp, #0xe - - mfsr $r12, $SP_USR - mfsr $r13, $IPC -#ifdef CONFIG_HWZOL - push_zol -#endif - movi $r17, -1 - move $r18, $r0 - mfsr $r19, $PSW - mfsr $r20, $IPSW - mfsr $r21, $P_IPSW - mfsr $r22, $P_IPC - mfsr $r23, $P_P0 - mfsr $r24, $P_P1 - smw.bim $r12, [$sp], $r24, #0 - addi $sp, $sp, -FUCOP_CTL_OFFSET - - /* Initialize kernel space $fp */ - andi $p0, $r20, #PSW_mskPOM - movi $p1, #0x0 - cmovz $fp, $p1, $p0 - - andi $r16, $r19, #PSW_mskINTL - slti $r17, $r16, #4 - bnez $r17, 1f - addi $r17, $r19, #-2 - mtsr $r17, $PSW - isb -1: - /* If it was superuser mode, we don't need to update $r25 */ - bnez $p0, 2f - la $p0, __entry_task - lw $r25, [$p0] -2: - .endm - - .text - -/* - * Exception Vector - */ -exception_handlers: - .long unhandled_exceptions !Reset/NMI - .long unhandled_exceptions !TLB fill - .long do_page_fault !PTE not present - .long do_dispatch_tlb_misc !TLB misc - .long unhandled_exceptions !TLB VLPT - .long unhandled_exceptions !Machine Error - .long do_debug_trap !Debug related - .long do_dispatch_general !General exception - .long eh_syscall !Syscall - .long asm_do_IRQ !IRQ - - skip_save_fucop_ctl -common_exception_handler: - save_user_regs - mfsr $p0, $ITYPE - andi $p0, $p0, #ITYPE_mskVECTOR - srli $p0, $p0, #ITYPE_offVECTOR - andi $p1, $p0, #NDS32_VECTOR_mskNONEXCEPTION - bnez $p1, 1f - sethi $lp, hi20(ret_from_exception) - ori $lp, $lp, lo12(ret_from_exception) - sethi $p1, hi20(exception_handlers) - ori $p1, $p1, lo12(exception_handlers) - lw $p1, [$p1+$p0<<2] - move $r0, $p0 - mfsr $r1, $EVA - mfsr $r2, $ITYPE - move $r3, $sp - mfsr $r4, $OIPC - /* enable gie if it is enabled in IPSW. */ - mfsr $r21, $PSW - andi $r20, $r20, #PSW_mskGIE /* r20 is $IPSW*/ - or $r21, $r21, $r20 - mtsr $r21, $PSW - dsb - jr $p1 - /* syscall */ -1: - addi $p1, $p0, #-NDS32_VECTOR_offEXCEPTION - bnez $p1, 2f - sethi $lp, hi20(ret_from_exception) - ori $lp, $lp, lo12(ret_from_exception) - sethi $p1, hi20(exception_handlers) - ori $p1, $p1, lo12(exception_handlers) - lwi $p1, [$p1+#NDS32_VECTOR_offEXCEPTION<<2] - jr $p1 - - /* interrupt */ -2: -#ifdef CONFIG_TRACE_IRQFLAGS - jal __trace_hardirqs_off -#endif - move $r0, $sp - sethi $lp, hi20(ret_from_intr) - ori $lp, $lp, lo12(ret_from_intr) - sethi $p0, hi20(exception_handlers) - ori $p0, $p0, lo12(exception_handlers) - lwi $p0, [$p0+#NDS32_VECTOR_offINTERRUPT<<2] - jr $p0 - - .macro EXCEPTION_VECTOR_DEBUG - .align 4 - mfsr $p0, $EDM_CTL - andi $p0, $p0, EDM_CTL_mskV3_EDM_MODE - tnez $p0, SWID_RAISE_INTERRUPT_LEVEL - .endm - - .macro EXCEPTION_VECTOR - .align 4 - sethi $p0, hi20(common_exception_handler) - ori $p0, $p0, lo12(common_exception_handler) - jral.ton $p0, $p0 - .endm - - .section ".text.init", #alloc, #execinstr - .global exception_vector -exception_vector: -.rept 6 - EXCEPTION_VECTOR -.endr - EXCEPTION_VECTOR_DEBUG -.rept 121 - EXCEPTION_VECTOR -.endr - .align 4 - .global exception_vector_end -exception_vector_end: diff --git a/arch/nds32/kernel/ex-exit.S b/arch/nds32/kernel/ex-exit.S deleted file mode 100644 index b30699911b81..000000000000 --- a/arch/nds32/kernel/ex-exit.S +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/unistd.h> -#include <asm/assembler.h> -#include <asm/nds32.h> -#include <asm/asm-offsets.h> -#include <asm/thread_info.h> -#include <asm/current.h> -#include <asm/fpu.h> - - - -#ifdef CONFIG_HWZOL - .macro pop_zol - mtusr $r14, $LB - mtusr $r15, $LE - mtusr $r16, $LC - .endm -#endif - - .macro restore_user_regs_first - setgie.d - isb -#if defined(CONFIG_FPU) - addi $sp, $sp, OSP_OFFSET - lmw.adm $r12, [$sp], $r25, #0x0 - sethi $p0, hi20(has_fpu) - lbsi $p0, [$p0+lo12(has_fpu)] - beqz $p0, 2f - mtsr $r25, $FUCOP_CTL -2: -#else - addi $sp, $sp, FUCOP_CTL_OFFSET - lmw.adm $r12, [$sp], $r24, #0x0 -#endif - mtsr $r12, $SP_USR - mtsr $r13, $IPC -#ifdef CONFIG_HWZOL - pop_zol -#endif - mtsr $r19, $PSW - mtsr $r20, $IPSW - mtsr $r21, $P_IPSW - mtsr $r22, $P_IPC - mtsr $r23, $P_P0 - mtsr $r24, $P_P1 - lmw.adm $sp, [$sp], $sp, #0xe - .endm - - .macro restore_user_regs_last - pop $p0 - cmovn $sp, $p0, $p0 - - iret - nop - - .endm - - .macro restore_user_regs - restore_user_regs_first - lmw.adm $r0, [$sp], $r25, #0x0 - addi $sp, $sp, OSP_OFFSET - restore_user_regs_last - .endm - - .macro fast_restore_user_regs - restore_user_regs_first - lmw.adm $r1, [$sp], $r25, #0x0 - addi $sp, $sp, OSP_OFFSET-4 - restore_user_regs_last - .endm - -#ifdef CONFIG_PREEMPTION - .macro preempt_stop - .endm -#else - .macro preempt_stop - setgie.d - isb - .endm -#define resume_kernel no_work_pending -#endif - -ENTRY(ret_from_exception) - preempt_stop -ENTRY(ret_from_intr) - -/* - * judge Kernel or user mode - * - */ - lwi $p0, [$sp+(#IPSW_OFFSET)] ! Check if in nested interrupt - andi $p0, $p0, #PSW_mskINTL - bnez $p0, resume_kernel ! done with iret - j resume_userspace - - -/* - * This is the fast syscall return path. We do as little as - * possible here, and this includes saving $r0 back into the SVC - * stack. - * fixed: tsk - $r25, syscall # - $r7, syscall table pointer - $r8 - */ -ENTRY(ret_fast_syscall) - gie_disable - lwi $r1, [tsk+#TSK_TI_FLAGS] - andi $p1, $r1, #_TIF_WORK_MASK - bnez $p1, fast_work_pending - fast_restore_user_regs ! iret - -/* - * Ok, we need to do extra processing, - * enter the slow path returning from syscall, while pending work. - */ -fast_work_pending: - swi $r0, [$sp+(#R0_OFFSET)] ! what is different from ret_from_exception -work_pending: - andi $p1, $r1, #_TIF_NEED_RESCHED - bnez $p1, work_resched - - andi $p1, $r1, #_TIF_SIGPENDING|#_TIF_NOTIFY_RESUME|#_TIF_NOTIFY_SIGNAL - beqz $p1, no_work_pending - - move $r0, $sp ! 'regs' - gie_enable - bal do_notify_resume - b ret_slow_syscall -work_resched: - bal schedule ! path, return to user mode - -/* - * "slow" syscall return path. - */ -ENTRY(resume_userspace) -ENTRY(ret_slow_syscall) - gie_disable - lwi $p0, [$sp+(#IPSW_OFFSET)] ! Check if in nested interrupt - andi $p0, $p0, #PSW_mskINTL - bnez $p0, no_work_pending ! done with iret - lwi $r1, [tsk+#TSK_TI_FLAGS] - andi $p1, $r1, #_TIF_WORK_MASK - bnez $p1, work_pending ! handle work_resched, sig_pend - -no_work_pending: -#ifdef CONFIG_TRACE_IRQFLAGS - lwi $p0, [$sp+(#IPSW_OFFSET)] - andi $p0, $p0, #0x1 - la $r10, __trace_hardirqs_off - la $r9, __trace_hardirqs_on - cmovz $r9, $p0, $r10 - jral $r9 -#endif - restore_user_regs ! return from iret - - -/* - * preemptive kernel - */ -#ifdef CONFIG_PREEMPTION -resume_kernel: - gie_disable - lwi $t0, [tsk+#TSK_TI_PREEMPT] - bnez $t0, no_work_pending - - lwi $t0, [tsk+#TSK_TI_FLAGS] - andi $p1, $t0, #_TIF_NEED_RESCHED - beqz $p1, no_work_pending - - lwi $t0, [$sp+(#IPSW_OFFSET)] ! Interrupts off? - andi $t0, $t0, #1 - beqz $t0, no_work_pending - - jal preempt_schedule_irq - b no_work_pending -#endif - -/* - * This is how we return from a fork. - */ -ENTRY(ret_from_fork) - bal schedule_tail - beqz $r6, 1f ! r6 stores fn for kernel thread - move $r0, $r7 ! prepare kernel thread arg - jral $r6 -1: - lwi $r1, [tsk+#TSK_TI_FLAGS] ! check for syscall tracing - andi $p1, $r1, #_TIF_WORK_SYSCALL_LEAVE ! are we tracing syscalls? - beqz $p1, ret_slow_syscall - move $r0, $sp - bal syscall_trace_leave - b ret_slow_syscall diff --git a/arch/nds32/kernel/ex-scall.S b/arch/nds32/kernel/ex-scall.S deleted file mode 100644 index 270050f1b7b1..000000000000 --- a/arch/nds32/kernel/ex-scall.S +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/unistd.h> -#include <asm/assembler.h> -#include <asm/nds32.h> -#include <asm/asm-offsets.h> -#include <asm/thread_info.h> -#include <asm/current.h> - -/* - * $r0 = previous task_struct, - * $r1 = next task_struct, - * previous and next are guaranteed not to be the same. - */ - -ENTRY(__switch_to) - - la $p0, __entry_task - sw $r1, [$p0] - addi $p1, $r0, #THREAD_CPU_CONTEXT - smw.bi $r6, [$p1], $r14, #0xb ! push r6~r14, fp, lp, sp - move $r25, $r1 -#if defined(CONFIG_FPU) - call _switch_fpu -#endif - addi $r1, $r25, #THREAD_CPU_CONTEXT - lmw.bi $r6, [$r1], $r14, #0xb ! pop r6~r14, fp, lp, sp - ret - - -#define tbl $r8 - -/* - * $r7 will be writen as syscall nr - */ - .macro get_scno - lwi $r7, [$sp + R15_OFFSET] - swi $r7, [$sp + SYSCALLNO_OFFSET] - .endm - - .macro updateipc - addi $r17, $r13, #4 ! $r13 is $IPC - swi $r17, [$sp + IPC_OFFSET] - .endm - -ENTRY(eh_syscall) - updateipc - - get_scno - gie_enable - - lwi $p0, [tsk+#TSK_TI_FLAGS] ! check for syscall tracing - - andi $p1, $p0, #_TIF_WORK_SYSCALL_ENTRY ! are we tracing syscalls? - bnez $p1, __sys_trace - - la $lp, ret_fast_syscall ! return address -jmp_systbl: - addi $p1, $r7, #-__NR_syscalls ! syscall number of syscall instruction is guarded by addembler - bgez $p1, _SCNO_EXCEED ! call sys_* routine - la tbl, sys_call_table ! load syscall table pointer - slli $p1, $r7, #2 - add $p1, tbl, $p1 - lwi $p1, [$p1] - jr $p1 ! no return - -_SCNO_EXCEED: - ori $r0, $r7, #0 - ori $r1, $sp, #0 - b bad_syscall - -/* - * This is the really slow path. We're going to be doing - * context switches, and waiting for our parent to respond. - */ -__sys_trace: - move $r0, $sp - bal syscall_trace_enter - move $r7, $r0 - la $lp, __sys_trace_return ! return address - - addi $p1, $r7, #1 - beqz $p1, ret_slow_syscall ! fatal signal is pending - - addi $p1, $sp, #R0_OFFSET ! pointer to regs - lmw.bi $r0, [$p1], $r5 ! have to reload $r0 - $r5 - b jmp_systbl - -__sys_trace_return: - swi $r0, [$sp+#R0_OFFSET] ! T: save returned $r0 - move $r0, $sp ! set pt_regs for syscall_trace_leave - bal syscall_trace_leave - b ret_slow_syscall - -ENTRY(sys_rt_sigreturn_wrapper) - addi $r0, $sp, #0 - b sys_rt_sigreturn -ENDPROC(sys_rt_sigreturn_wrapper) diff --git a/arch/nds32/kernel/fpu.c b/arch/nds32/kernel/fpu.c deleted file mode 100644 index 701c09a668de..000000000000 --- a/arch/nds32/kernel/fpu.c +++ /dev/null @@ -1,266 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <linux/sched.h> -#include <linux/signal.h> -#include <linux/sched/signal.h> -#include <asm/processor.h> -#include <asm/user.h> -#include <asm/io.h> -#include <asm/bitfield.h> -#include <asm/fpu.h> - -const struct fpu_struct init_fpuregs = { - .fd_regs = {[0 ... 31] = sNAN64}, - .fpcsr = FPCSR_INIT, -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) - .UDF_IEX_trap = 0 -#endif -}; - -void save_fpu(struct task_struct *tsk) -{ - unsigned int fpcfg, fpcsr; - - enable_fpu(); - fpcfg = ((__nds32__fmfcfg() & FPCFG_mskFREG) >> FPCFG_offFREG); - switch (fpcfg) { - case SP32_DP32_reg: - asm volatile ("fsdi $fd31, [%0+0xf8]\n\t" - "fsdi $fd30, [%0+0xf0]\n\t" - "fsdi $fd29, [%0+0xe8]\n\t" - "fsdi $fd28, [%0+0xe0]\n\t" - "fsdi $fd27, [%0+0xd8]\n\t" - "fsdi $fd26, [%0+0xd0]\n\t" - "fsdi $fd25, [%0+0xc8]\n\t" - "fsdi $fd24, [%0+0xc0]\n\t" - "fsdi $fd23, [%0+0xb8]\n\t" - "fsdi $fd22, [%0+0xb0]\n\t" - "fsdi $fd21, [%0+0xa8]\n\t" - "fsdi $fd20, [%0+0xa0]\n\t" - "fsdi $fd19, [%0+0x98]\n\t" - "fsdi $fd18, [%0+0x90]\n\t" - "fsdi $fd17, [%0+0x88]\n\t" - "fsdi $fd16, [%0+0x80]\n\t" - : /* no output */ - : "r" (&tsk->thread.fpu) - : "memory"); - fallthrough; - case SP32_DP16_reg: - asm volatile ("fsdi $fd15, [%0+0x78]\n\t" - "fsdi $fd14, [%0+0x70]\n\t" - "fsdi $fd13, [%0+0x68]\n\t" - "fsdi $fd12, [%0+0x60]\n\t" - "fsdi $fd11, [%0+0x58]\n\t" - "fsdi $fd10, [%0+0x50]\n\t" - "fsdi $fd9, [%0+0x48]\n\t" - "fsdi $fd8, [%0+0x40]\n\t" - : /* no output */ - : "r" (&tsk->thread.fpu) - : "memory"); - fallthrough; - case SP16_DP8_reg: - asm volatile ("fsdi $fd7, [%0+0x38]\n\t" - "fsdi $fd6, [%0+0x30]\n\t" - "fsdi $fd5, [%0+0x28]\n\t" - "fsdi $fd4, [%0+0x20]\n\t" - : /* no output */ - : "r" (&tsk->thread.fpu) - : "memory"); - fallthrough; - case SP8_DP4_reg: - asm volatile ("fsdi $fd3, [%1+0x18]\n\t" - "fsdi $fd2, [%1+0x10]\n\t" - "fsdi $fd1, [%1+0x8]\n\t" - "fsdi $fd0, [%1+0x0]\n\t" - "fmfcsr %0\n\t" - "swi %0, [%1+0x100]\n\t" - : "=&r" (fpcsr) - : "r"(&tsk->thread.fpu) - : "memory"); - } - disable_fpu(); -} - -void load_fpu(const struct fpu_struct *fpregs) -{ - unsigned int fpcfg, fpcsr; - - enable_fpu(); - fpcfg = ((__nds32__fmfcfg() & FPCFG_mskFREG) >> FPCFG_offFREG); - switch (fpcfg) { - case SP32_DP32_reg: - asm volatile ("fldi $fd31, [%0+0xf8]\n\t" - "fldi $fd30, [%0+0xf0]\n\t" - "fldi $fd29, [%0+0xe8]\n\t" - "fldi $fd28, [%0+0xe0]\n\t" - "fldi $fd27, [%0+0xd8]\n\t" - "fldi $fd26, [%0+0xd0]\n\t" - "fldi $fd25, [%0+0xc8]\n\t" - "fldi $fd24, [%0+0xc0]\n\t" - "fldi $fd23, [%0+0xb8]\n\t" - "fldi $fd22, [%0+0xb0]\n\t" - "fldi $fd21, [%0+0xa8]\n\t" - "fldi $fd20, [%0+0xa0]\n\t" - "fldi $fd19, [%0+0x98]\n\t" - "fldi $fd18, [%0+0x90]\n\t" - "fldi $fd17, [%0+0x88]\n\t" - "fldi $fd16, [%0+0x80]\n\t" - : /* no output */ - : "r" (fpregs)); - fallthrough; - case SP32_DP16_reg: - asm volatile ("fldi $fd15, [%0+0x78]\n\t" - "fldi $fd14, [%0+0x70]\n\t" - "fldi $fd13, [%0+0x68]\n\t" - "fldi $fd12, [%0+0x60]\n\t" - "fldi $fd11, [%0+0x58]\n\t" - "fldi $fd10, [%0+0x50]\n\t" - "fldi $fd9, [%0+0x48]\n\t" - "fldi $fd8, [%0+0x40]\n\t" - : /* no output */ - : "r" (fpregs)); - fallthrough; - case SP16_DP8_reg: - asm volatile ("fldi $fd7, [%0+0x38]\n\t" - "fldi $fd6, [%0+0x30]\n\t" - "fldi $fd5, [%0+0x28]\n\t" - "fldi $fd4, [%0+0x20]\n\t" - : /* no output */ - : "r" (fpregs)); - fallthrough; - case SP8_DP4_reg: - asm volatile ("fldi $fd3, [%1+0x18]\n\t" - "fldi $fd2, [%1+0x10]\n\t" - "fldi $fd1, [%1+0x8]\n\t" - "fldi $fd0, [%1+0x0]\n\t" - "lwi %0, [%1+0x100]\n\t" - "fmtcsr %0\n\t":"=&r" (fpcsr) - : "r"(fpregs)); - } - disable_fpu(); -} -void store_fpu_for_suspend(void) -{ -#ifdef CONFIG_LAZY_FPU - if (last_task_used_math != NULL) - save_fpu(last_task_used_math); - last_task_used_math = NULL; -#else - if (!used_math()) - return; - unlazy_fpu(current); -#endif - clear_fpu(task_pt_regs(current)); -} -inline void do_fpu_context_switch(struct pt_regs *regs) -{ - /* Enable to use FPU. */ - - if (!user_mode(regs)) { - pr_err("BUG: FPU is used in kernel mode.\n"); - BUG(); - return; - } - - enable_ptreg_fpu(regs); -#ifdef CONFIG_LAZY_FPU //Lazy FPU is used - if (last_task_used_math == current) - return; - if (last_task_used_math != NULL) - /* Other processes fpu state, save away */ - save_fpu(last_task_used_math); - last_task_used_math = current; -#endif - if (used_math()) { - load_fpu(¤t->thread.fpu); - } else { - /* First time FPU user. */ - load_fpu(&init_fpuregs); -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) - current->thread.fpu.UDF_IEX_trap = init_fpuregs.UDF_IEX_trap; -#endif - set_used_math(); - } - -} - -inline void fill_sigfpe_signo(unsigned int fpcsr, int *signo) -{ - if (fpcsr & FPCSR_mskOVFT) - *signo = FPE_FLTOVF; -#ifndef CONFIG_SUPPORT_DENORMAL_ARITHMETIC - else if (fpcsr & FPCSR_mskUDFT) - *signo = FPE_FLTUND; -#endif - else if (fpcsr & FPCSR_mskIVOT) - *signo = FPE_FLTINV; - else if (fpcsr & FPCSR_mskDBZT) - *signo = FPE_FLTDIV; - else if (fpcsr & FPCSR_mskIEXT) - *signo = FPE_FLTRES; -} - -inline void handle_fpu_exception(struct pt_regs *regs) -{ - unsigned int fpcsr; - int si_code = 0, si_signo = SIGFPE; -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) - unsigned long redo_except = FPCSR_mskDNIT|FPCSR_mskUDFT|FPCSR_mskIEXT; -#else - unsigned long redo_except = FPCSR_mskDNIT; -#endif - - lose_fpu(); - fpcsr = current->thread.fpu.fpcsr; - - if (fpcsr & redo_except) { - si_signo = do_fpuemu(regs, ¤t->thread.fpu); - fpcsr = current->thread.fpu.fpcsr; - if (!si_signo) { - current->thread.fpu.fpcsr &= ~(redo_except); - goto done; - } - } else if (fpcsr & FPCSR_mskRIT) { - if (!user_mode(regs)) - make_task_dead(SIGILL); - si_signo = SIGILL; - } - - switch (si_signo) { - case SIGFPE: - fill_sigfpe_signo(fpcsr, &si_code); - break; - case SIGILL: - show_regs(regs); - si_code = ILL_COPROC; - break; - case SIGBUS: - si_code = BUS_ADRERR; - break; - default: - break; - } - - force_sig_fault(si_signo, si_code, - (void __user *)instruction_pointer(regs)); -done: - own_fpu(); -} - -bool do_fpu_exception(unsigned int subtype, struct pt_regs *regs) -{ - int done = true; - /* Coprocessor disabled exception */ - if (subtype == FPU_DISABLE_EXCEPTION) { - preempt_disable(); - do_fpu_context_switch(regs); - preempt_enable(); - } - /* Coprocessor exception such as underflow and overflow */ - else if (subtype == FPU_EXCEPTION) - handle_fpu_exception(regs); - else - done = false; - return done; -} diff --git a/arch/nds32/kernel/ftrace.c b/arch/nds32/kernel/ftrace.c deleted file mode 100644 index 711bc8cd186d..000000000000 --- a/arch/nds32/kernel/ftrace.c +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/ftrace.h> -#include <linux/uaccess.h> -#include <asm/cacheflush.h> - -#ifndef CONFIG_DYNAMIC_FTRACE -extern void (*ftrace_trace_function)(unsigned long, unsigned long, - struct ftrace_ops*, struct ftrace_regs*); -extern void ftrace_graph_caller(void); - -noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip, - struct ftrace_ops *op, struct ftrace_regs *fregs) -{ - __asm__ (""); /* avoid to optimize as pure function */ -} - -noinline void _mcount(unsigned long parent_ip) -{ - /* save all state by the compiler prologue */ - - unsigned long ip = (unsigned long)__builtin_return_address(0); - - if (ftrace_trace_function != ftrace_stub) - ftrace_trace_function(ip - MCOUNT_INSN_SIZE, parent_ip, - NULL, NULL); - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - if (ftrace_graph_return != (trace_func_graph_ret_t)ftrace_stub - || ftrace_graph_entry != ftrace_graph_entry_stub) - ftrace_graph_caller(); -#endif - - /* restore all state by the compiler epilogue */ -} -EXPORT_SYMBOL(_mcount); - -#else /* CONFIG_DYNAMIC_FTRACE */ - -noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip, - struct ftrace_ops *op, struct ftrace_regs *fregs) -{ - __asm__ (""); /* avoid to optimize as pure function */ -} - -noinline void __naked _mcount(unsigned long parent_ip) -{ - __asm__ (""); /* avoid to optimize as pure function */ -} -EXPORT_SYMBOL(_mcount); - -#define XSTR(s) STR(s) -#define STR(s) #s -void _ftrace_caller(unsigned long parent_ip) -{ - /* save all state needed by the compiler prologue */ - - /* - * prepare arguments for real tracing function - * first arg : __builtin_return_address(0) - MCOUNT_INSN_SIZE - * second arg : parent_ip - */ - __asm__ __volatile__ ( - "move $r1, %0 \n\t" - "addi $r0, %1, #-" XSTR(MCOUNT_INSN_SIZE) "\n\t" - : - : "r" (parent_ip), "r" (__builtin_return_address(0))); - - /* a placeholder for the call to a real tracing function */ - __asm__ __volatile__ ( - "ftrace_call: \n\t" - "nop \n\t" - "nop \n\t" - "nop \n\t"); - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - /* a placeholder for the call to ftrace_graph_caller */ - __asm__ __volatile__ ( - "ftrace_graph_call: \n\t" - "nop \n\t" - "nop \n\t" - "nop \n\t"); -#endif - /* restore all state needed by the compiler epilogue */ -} - -static unsigned long gen_sethi_insn(unsigned long addr) -{ - unsigned long opcode = 0x46000000; - unsigned long imm = addr >> 12; - unsigned long rt_num = 0xf << 20; - - return ENDIAN_CONVERT(opcode | rt_num | imm); -} - -static unsigned long gen_ori_insn(unsigned long addr) -{ - unsigned long opcode = 0x58000000; - unsigned long imm = addr & 0x0000fff; - unsigned long rt_num = 0xf << 20; - unsigned long ra_num = 0xf << 15; - - return ENDIAN_CONVERT(opcode | rt_num | ra_num | imm); -} - -static unsigned long gen_jral_insn(unsigned long addr) -{ - unsigned long opcode = 0x4a000001; - unsigned long rt_num = 0x1e << 20; - unsigned long rb_num = 0xf << 10; - - return ENDIAN_CONVERT(opcode | rt_num | rb_num); -} - -static void ftrace_gen_call_insn(unsigned long *call_insns, - unsigned long addr) -{ - call_insns[0] = gen_sethi_insn(addr); /* sethi $r15, imm20u */ - call_insns[1] = gen_ori_insn(addr); /* ori $r15, $r15, imm15u */ - call_insns[2] = gen_jral_insn(addr); /* jral $lp, $r15 */ -} - -static int __ftrace_modify_code(unsigned long pc, unsigned long *old_insn, - unsigned long *new_insn, bool validate) -{ - unsigned long orig_insn[3]; - - if (validate) { - if (copy_from_kernel_nofault(orig_insn, (void *)pc, - MCOUNT_INSN_SIZE)) - return -EFAULT; - if (memcmp(orig_insn, old_insn, MCOUNT_INSN_SIZE)) - return -EINVAL; - } - - if (copy_to_kernel_nofault((void *)pc, new_insn, MCOUNT_INSN_SIZE)) - return -EPERM; - - return 0; -} - -static int ftrace_modify_code(unsigned long pc, unsigned long *old_insn, - unsigned long *new_insn, bool validate) -{ - int ret; - - ret = __ftrace_modify_code(pc, old_insn, new_insn, validate); - if (ret) - return ret; - - flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); - - return ret; -} - -int ftrace_update_ftrace_func(ftrace_func_t func) -{ - unsigned long pc = (unsigned long)&ftrace_call; - unsigned long old_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - unsigned long new_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - - if (func != ftrace_stub) - ftrace_gen_call_insn(new_insn, (unsigned long)func); - - return ftrace_modify_code(pc, old_insn, new_insn, false); -} - -int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) -{ - unsigned long pc = rec->ip; - unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - - ftrace_gen_call_insn(call_insn, addr); - - return ftrace_modify_code(pc, nop_insn, call_insn, true); -} - -int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, - unsigned long addr) -{ - unsigned long pc = rec->ip; - unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - - ftrace_gen_call_insn(call_insn, addr); - - return ftrace_modify_code(pc, call_insn, nop_insn, true); -} -#endif /* CONFIG_DYNAMIC_FTRACE */ - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER -void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, - unsigned long frame_pointer) -{ - unsigned long return_hooker = (unsigned long)&return_to_handler; - unsigned long old; - - if (unlikely(atomic_read(¤t->tracing_graph_pause))) - return; - - old = *parent; - - if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) - *parent = return_hooker; -} - -noinline void ftrace_graph_caller(void) -{ - unsigned long *parent_ip = - (unsigned long *)(__builtin_frame_address(2) - 4); - - unsigned long selfpc = - (unsigned long)(__builtin_return_address(1) - MCOUNT_INSN_SIZE); - - unsigned long frame_pointer = - (unsigned long)__builtin_frame_address(3); - - prepare_ftrace_return(parent_ip, selfpc, frame_pointer); -} - -extern unsigned long ftrace_return_to_handler(unsigned long frame_pointer); -void __naked return_to_handler(void) -{ - __asm__ __volatile__ ( - /* save state needed by the ABI */ - "smw.adm $r0,[$sp],$r1,#0x0 \n\t" - - /* get original return address */ - "move $r0, $fp \n\t" - "bal ftrace_return_to_handler\n\t" - "move $lp, $r0 \n\t" - - /* restore state needed by the ABI */ - "lmw.bim $r0,[$sp],$r1,#0x0 \n\t"); -} - -#ifdef CONFIG_DYNAMIC_FTRACE -extern unsigned long ftrace_graph_call; - -static int ftrace_modify_graph_caller(bool enable) -{ - unsigned long pc = (unsigned long)&ftrace_graph_call; - unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; - - ftrace_gen_call_insn(call_insn, (unsigned long)ftrace_graph_caller); - - if (enable) - return ftrace_modify_code(pc, nop_insn, call_insn, true); - else - return ftrace_modify_code(pc, call_insn, nop_insn, true); -} - -int ftrace_enable_ftrace_graph_caller(void) -{ - return ftrace_modify_graph_caller(true); -} - -int ftrace_disable_ftrace_graph_caller(void) -{ - return ftrace_modify_graph_caller(false); -} -#endif /* CONFIG_DYNAMIC_FTRACE */ - -#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ - - -#ifdef CONFIG_TRACE_IRQFLAGS -noinline void __trace_hardirqs_off(void) -{ - trace_hardirqs_off(); -} -noinline void __trace_hardirqs_on(void) -{ - trace_hardirqs_on(); -} -#endif /* CONFIG_TRACE_IRQFLAGS */ diff --git a/arch/nds32/kernel/head.S b/arch/nds32/kernel/head.S deleted file mode 100644 index 7347f00451a9..000000000000 --- a/arch/nds32/kernel/head.S +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <linux/init.h> -#include <linux/pgtable.h> -#include <asm/ptrace.h> -#include <asm/asm-offsets.h> -#include <asm/page.h> -#include <linux/sizes.h> -#include <asm/thread_info.h> - -#ifdef CONFIG_CPU_BIG_ENDIAN -#define OF_DT_MAGIC 0xd00dfeed -#else -#define OF_DT_MAGIC 0xedfe0dd0 -#endif - - .globl swapper_pg_dir - .equ swapper_pg_dir, TEXTADDR - 0x4000 - -/* - * Kernel startup entry point. - */ - .section ".head.text", "ax" - .type _stext, %function -ENTRY(_stext) - setgie.d ! Disable interrupt - isb -/* - * Disable I/D-cache and enable it at a proper time - */ - mfsr $r0, $mr8 - li $r1, #~(CACHE_CTL_mskIC_EN|CACHE_CTL_mskDC_EN) - and $r0, $r0, $r1 - mtsr $r0, $mr8 - -/* - * Process device tree blob - */ - andi $r0,$r2,#0x3 - li $r10, 0 - bne $r0, $r10, _nodtb - lwi $r0, [$r2] - li $r1, OF_DT_MAGIC - bne $r0, $r1, _nodtb - move $r10, $r2 -_nodtb: - -/* - * Create a temporary mapping area for booting, before start_kernel - */ - sethi $r4, hi20(swapper_pg_dir) - li $p0, (PAGE_OFFSET - PHYS_OFFSET) - sub $r4, $r4, $p0 - tlbop FlushAll ! invalidate TLB\n" - isb - mtsr $r4, $L1_PPTB ! load page table pointer\n" - -#ifdef CONFIG_CPU_DCACHE_DISABLE - #define MMU_CTL_NTCC MMU_CTL_CACHEABLE_NON -#else - #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH - #define MMU_CTL_NTCC MMU_CTL_CACHEABLE_WT - #else - #define MMU_CTL_NTCC MMU_CTL_CACHEABLE_WB - #endif -#endif - -/* set NTC cacheability, mutliple page size in use */ - mfsr $r3, $MMU_CTL -#if CONFIG_MEMORY_START >= 0xc0000000 - ori $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC3) -#elif CONFIG_MEMORY_START >= 0x80000000 - ori $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC2) -#elif CONFIG_MEMORY_START >= 0x40000000 - ori $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC1) -#else - ori $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC0) -#endif - -#ifdef CONFIG_ANDES_PAGE_SIZE_4KB - ori $r3, $r3, #(MMU_CTL_mskMPZIU) -#else - ori $r3, $r3, #(MMU_CTL_mskMPZIU|MMU_CTL_D8KB) -#endif -#ifdef CONFIG_HW_SUPPORT_UNALIGNMENT_ACCESS - li $r0, #MMU_CTL_UNA - or $r3, $r3, $r0 -#endif - mtsr $r3, $MMU_CTL - isb - -/* set page size and size of kernel image */ - mfsr $r0, $MMU_CFG - srli $r3, $r0, MMU_CFG_offfEPSZ - zeb $r3, $r3 - bnez $r3, _extra_page_size_support -#ifdef CONFIG_ANDES_PAGE_SIZE_4KB - li $r5, #SZ_4K ! Use 4KB page size -#else - li $r5, #SZ_8K ! Use 8KB page size - li $r3, #1 -#endif - mtsr $r3, $TLB_MISC - b _image_size_check - -_extra_page_size_support: ! Use epzs pages size - clz $r6, $r3 - subri $r2, $r6, #31 - li $r3, #1 - sll $r3, $r3, $r2 - /* MMU_CFG.EPSZ value -> meaning */ - mul $r5, $r3, $r3 - slli $r5, $r5, #14 - /* MMU_CFG.EPSZ -> TLB_MISC.ACC_PSZ */ - addi $r3, $r2, #0x2 - mtsr $r3, $TLB_MISC - -_image_size_check: - /* calculate the image maximum size accepted by TLB config */ - andi $r6, $r0, MMU_CFG_mskTBW - andi $r0, $r0, MMU_CFG_mskTBS - srli $r6, $r6, MMU_CFG_offTBW - srli $r0, $r0, MMU_CFG_offTBS - addi $r6, $r6, #0x1 ! MMU_CFG.TBW value -> meaning - addi $r0, $r0, #0x2 ! MMU_CFG.TBS value -> meaning - sll $r0, $r6, $r0 ! entries = k-way * n-set - mul $r6, $r0, $r5 ! max size = entries * page size - /* check kernel image size */ - la $r3, (_end - PAGE_OFFSET) - bgt $r3, $r6, __error - - li $r2, #(PHYS_OFFSET + TLB_DATA_kernel_text_attr) - li $r3, PAGE_OFFSET - add $r6, $r6, $r3 - -_tlb: - mtsr $r3, $TLB_VPN - dsb - tlbop $r2, RWR - isb - add $r3, $r3, $r5 - add $r2, $r2, $r5 - bgt $r6, $r3, _tlb - mfsr $r3, $TLB_MISC ! setup access page size - li $r2, #~0xf - and $r3, $r3, $r2 -#ifdef CONFIG_ANDES_PAGE_SIZE_8KB - ori $r3, $r3, #0x1 -#endif - mtsr $r3, $TLB_MISC - - mfsr $r0, $MISC_CTL ! Enable BTB, RTP, shadow sp, and HW_PRE - ori $r0, $r0, #MISC_init - mtsr $r0, $MISC_CTL - - mfsr $p1, $PSW - li $r15, #~PSW_clr ! clear WBNA|DME|IME|DT|IT|POM|INTL|GIE - and $p1, $p1, $r15 - ori $p1, $p1, #PSW_init - mtsr $p1, $IPSW ! when iret, it will automatically enable MMU - la $lp, __mmap_switched - mtsr $lp, $IPC - iret - nop - - .type __switch_data, %object -__switch_data: - .long __bss_start ! $r6 - .long _end ! $r7 - .long __atags_pointer ! $atag_pointer - .long init_task ! $r9, move to $r25 - .long init_thread_union + THREAD_SIZE ! $sp - - -/* - * The following fragment of code is executed with the MMU on in MMU mode, - * and uses absolute addresses; this is not position independent. - */ - .align - .type __mmap_switched, %function -__mmap_switched: - la $r3, __switch_data - lmw.bim $r6, [$r3], $r9, #0b0001 - move $r25, $r9 - move $fp, #0 ! Clear BSS (and zero $fp) - beq $r7, $r6, _RRT -1: swi.bi $fp, [$r6], #4 - bne $r7, $r6, 1b - swi $r10, [$r8] - -_RRT: - b start_kernel - -__error: - b __error diff --git a/arch/nds32/kernel/irq.c b/arch/nds32/kernel/irq.c deleted file mode 100644 index 6ff5a672be27..000000000000 --- a/arch/nds32/kernel/irq.c +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/irqchip.h> - -void __init init_IRQ(void) -{ - irqchip_init(); -} diff --git a/arch/nds32/kernel/module.c b/arch/nds32/kernel/module.c deleted file mode 100644 index 3897fd14a21d..000000000000 --- a/arch/nds32/kernel/module.c +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/module.h> -#include <linux/elf.h> -#include <linux/vmalloc.h> -#include <linux/moduleloader.h> -#include <linux/pgtable.h> - -void *module_alloc(unsigned long size) -{ - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); -} - -void module_free(struct module *module, void *region) -{ - vfree(region); -} - -int module_frob_arch_sections(Elf_Ehdr * hdr, - Elf_Shdr * sechdrs, - char *secstrings, struct module *mod) -{ - return 0; -} - -void do_reloc16(unsigned int val, unsigned int *loc, unsigned int val_mask, - unsigned int val_shift, unsigned int loc_mask, - unsigned int partial_in_place, unsigned int swap) -{ - unsigned int tmp = 0, tmp2 = 0; - - __asm__ __volatile__("\tlhi.bi\t%0, [%2], 0\n" - "\tbeqz\t%3, 1f\n" - "\twsbh\t%0, %1\n" - "1:\n":"=r"(tmp):"0"(tmp), "r"(loc), "r"(swap) - ); - - tmp2 = tmp & loc_mask; - if (partial_in_place) { - tmp &= (~loc_mask); - tmp = - tmp2 | ((tmp + ((val & val_mask) >> val_shift)) & val_mask); - } else { - tmp = tmp2 | ((val & val_mask) >> val_shift); - } - - __asm__ __volatile__("\tbeqz\t%3, 2f\n" - "\twsbh\t%0, %1\n" - "2:\n" - "\tshi.bi\t%0, [%2], 0\n":"=r"(tmp):"0"(tmp), - "r"(loc), "r"(swap) - ); -} - -void do_reloc32(unsigned int val, unsigned int *loc, unsigned int val_mask, - unsigned int val_shift, unsigned int loc_mask, - unsigned int partial_in_place, unsigned int swap) -{ - unsigned int tmp = 0, tmp2 = 0; - - __asm__ __volatile__("\tlmw.bi\t%0, [%2], %0, 0\n" - "\tbeqz\t%3, 1f\n" - "\twsbh\t%0, %1\n" - "\trotri\t%0, %1, 16\n" - "1:\n":"=r"(tmp):"0"(tmp), "r"(loc), "r"(swap) - ); - - tmp2 = tmp & loc_mask; - if (partial_in_place) { - tmp &= (~loc_mask); - tmp = - tmp2 | ((tmp + ((val & val_mask) >> val_shift)) & val_mask); - } else { - tmp = tmp2 | ((val & val_mask) >> val_shift); - } - - __asm__ __volatile__("\tbeqz\t%3, 2f\n" - "\twsbh\t%0, %1\n" - "\trotri\t%0, %1, 16\n" - "2:\n" - "\tsmw.bi\t%0, [%2], %0, 0\n":"=r"(tmp):"0"(tmp), - "r"(loc), "r"(swap) - ); -} - -static inline int exceed_limit(int offset, unsigned int val_mask, - struct module *module, Elf32_Rela * rel, - unsigned int relindex, unsigned int reloc_order) -{ - int abs_off = offset < 0 ? ~offset : offset; - - if (abs_off & (~val_mask)) { - pr_err("\n%s: relocation type %d out of range.\n" - "please rebuild the kernel module with gcc option \"-Wa,-mno-small-text\".\n", - module->name, ELF32_R_TYPE(rel->r_info)); - pr_err("section %d reloc %d offset 0x%x relative 0x%x.\n", - relindex, reloc_order, rel->r_offset, offset); - return true; - } - return false; -} - -#ifdef __NDS32_EL__ -#define NEED_SWAP 1 -#else -#define NEED_SWAP 0 -#endif - -int -apply_relocate_add(Elf32_Shdr * sechdrs, const char *strtab, - unsigned int symindex, unsigned int relindex, - struct module *module) -{ - Elf32_Shdr *symsec = sechdrs + symindex; - Elf32_Shdr *relsec = sechdrs + relindex; - Elf32_Shdr *dstsec = sechdrs + relsec->sh_info; - Elf32_Rela *rel = (void *)relsec->sh_addr; - unsigned int i; - - for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rela); i++, rel++) { - Elf32_Addr *loc; - Elf32_Sym *sym; - Elf32_Addr v; - s32 offset; - - offset = ELF32_R_SYM(rel->r_info); - if (offset < 0 - || offset > (symsec->sh_size / sizeof(Elf32_Sym))) { - pr_err("%s: bad relocation\n", module->name); - pr_err("section %d reloc %d\n", relindex, i); - return -ENOEXEC; - } - - sym = ((Elf32_Sym *) symsec->sh_addr) + offset; - - if (rel->r_offset < 0 - || rel->r_offset > dstsec->sh_size - sizeof(u16)) { - pr_err("%s: out of bounds relocation\n", module->name); - pr_err("section %d reloc %d offset 0x%0x size %d\n", - relindex, i, rel->r_offset, dstsec->sh_size); - return -ENOEXEC; - } - - loc = (Elf32_Addr *) (dstsec->sh_addr + rel->r_offset); - v = sym->st_value + rel->r_addend; - - switch (ELF32_R_TYPE(rel->r_info)) { - case R_NDS32_NONE: - case R_NDS32_INSN16: - case R_NDS32_LABEL: - case R_NDS32_LONGCALL1: - case R_NDS32_LONGCALL2: - case R_NDS32_LONGCALL3: - case R_NDS32_LONGCALL4: - case R_NDS32_LONGJUMP1: - case R_NDS32_LONGJUMP2: - case R_NDS32_LONGJUMP3: - case R_NDS32_9_FIXED_RELA: - case R_NDS32_15_FIXED_RELA: - case R_NDS32_17_FIXED_RELA: - case R_NDS32_25_FIXED_RELA: - case R_NDS32_LOADSTORE: - case R_NDS32_DWARF2_OP1_RELA: - case R_NDS32_DWARF2_OP2_RELA: - case R_NDS32_DWARF2_LEB_RELA: - case R_NDS32_RELA_NOP_MIX ... R_NDS32_RELA_NOP_MAX: - break; - - case R_NDS32_32_RELA: - do_reloc32(v, loc, 0xffffffff, 0, 0, 0, 0); - break; - - case R_NDS32_HI20_RELA: - do_reloc32(v, loc, 0xfffff000, 12, 0xfff00000, 0, - NEED_SWAP); - break; - - case R_NDS32_LO12S3_RELA: - do_reloc32(v, loc, 0x00000fff, 3, 0xfffff000, 0, - NEED_SWAP); - break; - - case R_NDS32_LO12S2_RELA: - do_reloc32(v, loc, 0x00000fff, 2, 0xfffff000, 0, - NEED_SWAP); - break; - - case R_NDS32_LO12S1_RELA: - do_reloc32(v, loc, 0x00000fff, 1, 0xfffff000, 0, - NEED_SWAP); - break; - - case R_NDS32_LO12S0_RELA: - case R_NDS32_LO12S0_ORI_RELA: - do_reloc32(v, loc, 0x00000fff, 0, 0xfffff000, 0, - NEED_SWAP); - break; - - case R_NDS32_9_PCREL_RELA: - if (exceed_limit - ((v - (Elf32_Addr) loc), 0x000000ff, module, rel, - relindex, i)) - return -ENOEXEC; - do_reloc16(v - (Elf32_Addr) loc, loc, 0x000001ff, 1, - 0xffffff00, 0, NEED_SWAP); - break; - - case R_NDS32_15_PCREL_RELA: - if (exceed_limit - ((v - (Elf32_Addr) loc), 0x00003fff, module, rel, - relindex, i)) - return -ENOEXEC; - do_reloc32(v - (Elf32_Addr) loc, loc, 0x00007fff, 1, - 0xffffc000, 0, NEED_SWAP); - break; - - case R_NDS32_17_PCREL_RELA: - if (exceed_limit - ((v - (Elf32_Addr) loc), 0x0000ffff, module, rel, - relindex, i)) - return -ENOEXEC; - do_reloc32(v - (Elf32_Addr) loc, loc, 0x0001ffff, 1, - 0xffff0000, 0, NEED_SWAP); - break; - - case R_NDS32_25_PCREL_RELA: - if (exceed_limit - ((v - (Elf32_Addr) loc), 0x00ffffff, module, rel, - relindex, i)) - return -ENOEXEC; - do_reloc32(v - (Elf32_Addr) loc, loc, 0x01ffffff, 1, - 0xff000000, 0, NEED_SWAP); - break; - case R_NDS32_WORD_9_PCREL_RELA: - if (exceed_limit - ((v - (Elf32_Addr) loc), 0x000000ff, module, rel, - relindex, i)) - return -ENOEXEC; - do_reloc32(v - (Elf32_Addr) loc, loc, 0x000001ff, 1, - 0xffffff00, 0, NEED_SWAP); - break; - - case R_NDS32_SDA15S3_RELA: - case R_NDS32_SDA15S2_RELA: - case R_NDS32_SDA15S1_RELA: - case R_NDS32_SDA15S0_RELA: - pr_err("%s: unsupported relocation type %d.\n", - module->name, ELF32_R_TYPE(rel->r_info)); - pr_err - ("Small data section access doesn't work in the kernel space; " - "please rebuild the kernel module with gcc option -mcmodel=large.\n"); - pr_err("section %d reloc %d offset 0x%x size %d\n", - relindex, i, rel->r_offset, dstsec->sh_size); - break; - - default: - pr_err("%s: unsupported relocation type %d.\n", - module->name, ELF32_R_TYPE(rel->r_info)); - pr_err("section %d reloc %d offset 0x%x size %d\n", - relindex, i, rel->r_offset, dstsec->sh_size); - } - } - return 0; -} - -int -module_finalize(const Elf32_Ehdr * hdr, const Elf_Shdr * sechdrs, - struct module *module) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/nds32/kernel/nds32_ksyms.c b/arch/nds32/kernel/nds32_ksyms.c deleted file mode 100644 index 20719e42ae36..000000000000 --- a/arch/nds32/kernel/nds32_ksyms.c +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/module.h> -#include <linux/string.h> -#include <linux/delay.h> -#include <linux/in6.h> -#include <linux/syscalls.h> -#include <linux/uaccess.h> - -#include <asm/checksum.h> -#include <asm/io.h> -#include <asm/ftrace.h> -#include <asm/proc-fns.h> - -/* mem functions */ -EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(memmove); -EXPORT_SYMBOL(memzero); - -/* user mem (segment) */ -EXPORT_SYMBOL(__arch_copy_from_user); -EXPORT_SYMBOL(__arch_copy_to_user); -EXPORT_SYMBOL(__arch_clear_user); diff --git a/arch/nds32/kernel/perf_event_cpu.c b/arch/nds32/kernel/perf_event_cpu.c deleted file mode 100644 index a78a879e7ef1..000000000000 --- a/arch/nds32/kernel/perf_event_cpu.c +++ /dev/null @@ -1,1500 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2008-2017 Andes Technology Corporation - * - * Reference ARMv7: Jean Pihet <jpihet@mvista.com> - * 2010 (c) MontaVista Software, LLC. - */ - -#include <linux/perf_event.h> -#include <linux/bitmap.h> -#include <linux/export.h> -#include <linux/kernel.h> -#include <linux/of.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/spinlock.h> -#include <linux/pm_runtime.h> -#include <linux/ftrace.h> -#include <linux/uaccess.h> -#include <linux/sched/clock.h> -#include <linux/percpu-defs.h> - -#include <asm/pmu.h> -#include <asm/irq_regs.h> -#include <asm/nds32.h> -#include <asm/stacktrace.h> -#include <asm/perf_event.h> -#include <nds32_intrinsic.h> - -/* Set at runtime when we know what CPU type we are. */ -static struct nds32_pmu *cpu_pmu; - -static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events); -static void nds32_pmu_start(struct nds32_pmu *cpu_pmu); -static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu); -static struct platform_device_id cpu_pmu_plat_device_ids[] = { - {.name = "nds32-pfm"}, - {}, -}; - -static int nds32_pmu_map_cache_event(const unsigned int (*cache_map) - [PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX], u64 config) -{ - unsigned int cache_type, cache_op, cache_result, ret; - - cache_type = (config >> 0) & 0xff; - if (cache_type >= PERF_COUNT_HW_CACHE_MAX) - return -EINVAL; - - cache_op = (config >> 8) & 0xff; - if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) - return -EINVAL; - - cache_result = (config >> 16) & 0xff; - if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) - return -EINVAL; - - ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; - - if (ret == CACHE_OP_UNSUPPORTED) - return -ENOENT; - - return ret; -} - -static int -nds32_pmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX], - u64 config) -{ - int mapping; - - if (config >= PERF_COUNT_HW_MAX) - return -ENOENT; - - mapping = (*event_map)[config]; - return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; -} - -static int nds32_pmu_map_raw_event(u32 raw_event_mask, u64 config) -{ - int ev_type = (int)(config & raw_event_mask); - int idx = config >> 8; - - switch (idx) { - case 0: - ev_type = PFM_OFFSET_MAGIC_0 + ev_type; - if (ev_type >= SPAV3_0_SEL_LAST || ev_type <= SPAV3_0_SEL_BASE) - return -ENOENT; - break; - case 1: - ev_type = PFM_OFFSET_MAGIC_1 + ev_type; - if (ev_type >= SPAV3_1_SEL_LAST || ev_type <= SPAV3_1_SEL_BASE) - return -ENOENT; - break; - case 2: - ev_type = PFM_OFFSET_MAGIC_2 + ev_type; - if (ev_type >= SPAV3_2_SEL_LAST || ev_type <= SPAV3_2_SEL_BASE) - return -ENOENT; - break; - default: - return -ENOENT; - } - - return ev_type; -} - -int -nds32_pmu_map_event(struct perf_event *event, - const unsigned int (*event_map)[PERF_COUNT_HW_MAX], - const unsigned int (*cache_map) - [PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask) -{ - u64 config = event->attr.config; - - switch (event->attr.type) { - case PERF_TYPE_HARDWARE: - return nds32_pmu_map_hw_event(event_map, config); - case PERF_TYPE_HW_CACHE: - return nds32_pmu_map_cache_event(cache_map, config); - case PERF_TYPE_RAW: - return nds32_pmu_map_raw_event(raw_event_mask, config); - } - - return -ENOENT; -} - -static int nds32_spav3_map_event(struct perf_event *event) -{ - return nds32_pmu_map_event(event, &nds32_pfm_perf_map, - &nds32_pfm_perf_cache_map, SOFTWARE_EVENT_MASK); -} - -static inline u32 nds32_pfm_getreset_flags(void) -{ - /* Read overflow status */ - u32 val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 old_val = val; - - /* Write overflow bit to clear status, and others keep it 0 */ - u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; - - __nds32__mtsr(val | ov_flag, NDS32_SR_PFM_CTL); - - return old_val; -} - -static inline int nds32_pfm_has_overflowed(u32 pfm) -{ - u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; - - return pfm & ov_flag; -} - -static inline int nds32_pfm_counter_has_overflowed(u32 pfm, int idx) -{ - u32 mask = 0; - - switch (idx) { - case 0: - mask = PFM_CTL_OVF[0]; - break; - case 1: - mask = PFM_CTL_OVF[1]; - break; - case 2: - mask = PFM_CTL_OVF[2]; - break; - default: - pr_err("%s index wrong\n", __func__); - break; - } - return pfm & mask; -} - -/* - * Set the next IRQ period, based on the hwc->period_left value. - * To be called with the event disabled in hw: - */ -int nds32_pmu_event_set_period(struct perf_event *event) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - s64 left = local64_read(&hwc->period_left); - s64 period = hwc->sample_period; - int ret = 0; - - /* The period may have been changed by PERF_EVENT_IOC_PERIOD */ - if (unlikely(period != hwc->last_period)) - left = period - (hwc->last_period - left); - - if (unlikely(left <= -period)) { - left = period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (unlikely(left <= 0)) { - left += period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (left > (s64)nds32_pmu->max_period) - left = nds32_pmu->max_period; - - /* - * The hw event starts counting from this event offset, - * mark it to be able to extract future "deltas": - */ - local64_set(&hwc->prev_count, (u64)(-left)); - - nds32_pmu->write_counter(event, (u64)(-left) & nds32_pmu->max_period); - - perf_event_update_userpage(event); - - return ret; -} - -static irqreturn_t nds32_pmu_handle_irq(int irq_num, void *dev) -{ - u32 pfm; - struct perf_sample_data data; - struct nds32_pmu *cpu_pmu = (struct nds32_pmu *)dev; - struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events(); - struct pt_regs *regs; - int idx; - /* - * Get and reset the IRQ flags - */ - pfm = nds32_pfm_getreset_flags(); - - /* - * Did an overflow occur? - */ - if (!nds32_pfm_has_overflowed(pfm)) - return IRQ_NONE; - - /* - * Handle the counter(s) overflow(s) - */ - regs = get_irq_regs(); - - nds32_pmu_stop(cpu_pmu); - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - /* Ignore if we don't have an event. */ - if (!event) - continue; - - /* - * We have a single interrupt for all counters. Check that - * each counter has overflowed before we process it. - */ - if (!nds32_pfm_counter_has_overflowed(pfm, idx)) - continue; - - hwc = &event->hw; - nds32_pmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!nds32_pmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - nds32_pmu_start(cpu_pmu); - /* - * Handle the pending perf events. - * - * Note: this call *must* be run with interrupts disabled. For - * platforms that can have the PMU interrupts raised as an NMI, this - * will not work. - */ - irq_work_run(); - - return IRQ_HANDLED; -} - -static inline int nds32_pfm_counter_valid(struct nds32_pmu *cpu_pmu, int idx) -{ - return ((idx >= 0) && (idx < cpu_pmu->num_events)); -} - -static inline int nds32_pfm_disable_counter(int idx) -{ - unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 mask = 0; - - mask = PFM_CTL_EN[idx]; - val &= ~mask; - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - return idx; -} - -/* - * Add an event filter to a given event. - */ -static int nds32_pmu_set_event_filter(struct hw_perf_event *event, - struct perf_event_attr *attr) -{ - unsigned long config_base = 0; - int idx = event->idx; - unsigned long no_kernel_tracing = 0; - unsigned long no_user_tracing = 0; - /* If index is -1, do not do anything */ - if (idx == -1) - return 0; - - no_kernel_tracing = PFM_CTL_KS[idx]; - no_user_tracing = PFM_CTL_KU[idx]; - /* - * Default: enable both kernel and user mode tracing. - */ - if (attr->exclude_user) - config_base |= no_user_tracing; - - if (attr->exclude_kernel) - config_base |= no_kernel_tracing; - - /* - * Install the filter into config_base as this is used to - * construct the event type. - */ - event->config_base |= config_base; - return 0; -} - -static inline void nds32_pfm_write_evtsel(int idx, u32 evnum) -{ - u32 offset = 0; - u32 ori_val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 ev_mask = 0; - u32 no_kernel_mask = 0; - u32 no_user_mask = 0; - u32 val; - - offset = PFM_CTL_OFFSEL[idx]; - /* Clear previous mode selection, and write new one */ - no_kernel_mask = PFM_CTL_KS[idx]; - no_user_mask = PFM_CTL_KU[idx]; - ori_val &= ~no_kernel_mask; - ori_val &= ~no_user_mask; - if (evnum & no_kernel_mask) - ori_val |= no_kernel_mask; - - if (evnum & no_user_mask) - ori_val |= no_user_mask; - - /* Clear previous event selection */ - ev_mask = PFM_CTL_SEL[idx]; - ori_val &= ~ev_mask; - evnum &= SOFTWARE_EVENT_MASK; - - /* undo the linear mapping */ - evnum = get_converted_evet_hw_num(evnum); - val = ori_val | (evnum << offset); - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); -} - -static inline int nds32_pfm_enable_counter(int idx) -{ - unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 mask = 0; - - mask = PFM_CTL_EN[idx]; - val |= mask; - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - return idx; -} - -static inline int nds32_pfm_enable_intens(int idx) -{ - unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 mask = 0; - - mask = PFM_CTL_IE[idx]; - val |= mask; - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - return idx; -} - -static inline int nds32_pfm_disable_intens(int idx) -{ - unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); - u32 mask = 0; - - mask = PFM_CTL_IE[idx]; - val &= ~mask; - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - return idx; -} - -static int event_requires_mode_exclusion(struct perf_event_attr *attr) -{ - /* Other modes NDS32 does not support */ - return attr->exclude_user || attr->exclude_kernel; -} - -static void nds32_pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - unsigned int evnum = 0; - struct hw_perf_event *hwc = &event->hw; - struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); - struct pmu_hw_events *events = cpu_pmu->get_hw_events(); - int idx = hwc->idx; - - if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { - pr_err("CPU enabling wrong pfm counter IRQ enable\n"); - return; - } - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - nds32_pfm_disable_counter(idx); - - /* - * Check whether we need to exclude the counter from certain modes. - */ - if ((!cpu_pmu->set_event_filter || - cpu_pmu->set_event_filter(hwc, &event->attr)) && - event_requires_mode_exclusion(&event->attr)) { - pr_notice - ("NDS32 performance counters do not support mode exclusion\n"); - hwc->config_base = 0; - } - /* Write event */ - evnum = hwc->config_base; - nds32_pfm_write_evtsel(idx, evnum); - - /* - * Enable interrupt for this counter - */ - nds32_pfm_enable_intens(idx); - - /* - * Enable counter - */ - nds32_pfm_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void nds32_pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); - struct pmu_hw_events *events = cpu_pmu->get_hw_events(); - int idx = hwc->idx; - - if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { - pr_err("CPU disabling wrong pfm counter IRQ enable %d\n", idx); - return; - } - - /* - * Disable counter and interrupt - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - nds32_pfm_disable_counter(idx); - - /* - * Disable interrupt for this counter - */ - nds32_pfm_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static inline u32 nds32_pmu_read_counter(struct perf_event *event) -{ - struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - u32 count = 0; - - if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { - pr_err("CPU reading wrong counter %d\n", idx); - } else { - switch (idx) { - case PFMC0: - count = __nds32__mfsr(NDS32_SR_PFMC0); - break; - case PFMC1: - count = __nds32__mfsr(NDS32_SR_PFMC1); - break; - case PFMC2: - count = __nds32__mfsr(NDS32_SR_PFMC2); - break; - default: - pr_err - ("%s: CPU has no performance counters %d\n", - __func__, idx); - } - } - return count; -} - -static inline void nds32_pmu_write_counter(struct perf_event *event, u32 value) -{ - struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { - pr_err("CPU writing wrong counter %d\n", idx); - } else { - switch (idx) { - case PFMC0: - __nds32__mtsr_isb(value, NDS32_SR_PFMC0); - break; - case PFMC1: - __nds32__mtsr_isb(value, NDS32_SR_PFMC1); - break; - case PFMC2: - __nds32__mtsr_isb(value, NDS32_SR_PFMC2); - break; - default: - pr_err - ("%s: CPU has no performance counters %d\n", - __func__, idx); - } - } -} - -static int nds32_pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - struct hw_perf_event *hwc = &event->hw; - /* - * Current implementation maps cycles, instruction count and cache-miss - * to specific counter. - * However, multiple of the 3 counters are able to count these events. - * - * - * SOFTWARE_EVENT_MASK mask for getting event num , - * This is defined by Jia-Rung, you can change the polocies. - * However, do not exceed 8 bits. This is hardware specific. - * The last number is SPAv3_2_SEL_LAST. - */ - unsigned long evtype = hwc->config_base & SOFTWARE_EVENT_MASK; - - idx = get_converted_event_idx(evtype); - /* - * Try to get the counter for correpsonding event - */ - if (evtype == SPAV3_0_SEL_TOTAL_CYCLES) { - if (!test_and_set_bit(idx, cpuc->used_mask)) - return idx; - if (!test_and_set_bit(NDS32_IDX_COUNTER0, cpuc->used_mask)) - return NDS32_IDX_COUNTER0; - if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) - return NDS32_IDX_COUNTER1; - } else if (evtype == SPAV3_1_SEL_COMPLETED_INSTRUCTION) { - if (!test_and_set_bit(idx, cpuc->used_mask)) - return idx; - else if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) - return NDS32_IDX_COUNTER1; - else if (!test_and_set_bit - (NDS32_IDX_CYCLE_COUNTER, cpuc->used_mask)) - return NDS32_IDX_CYCLE_COUNTER; - } else { - if (!test_and_set_bit(idx, cpuc->used_mask)) - return idx; - } - return -EAGAIN; -} - -static void nds32_pmu_start(struct nds32_pmu *cpu_pmu) -{ - unsigned long flags; - unsigned int val; - struct pmu_hw_events *events = cpu_pmu->get_hw_events(); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Enable all counters , NDS PFM has 3 counters */ - val = __nds32__mfsr(NDS32_SR_PFM_CTL); - val |= (PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu) -{ - unsigned long flags; - unsigned int val; - struct pmu_hw_events *events = cpu_pmu->get_hw_events(); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable all counters , NDS PFM has 3 counters */ - val = __nds32__mfsr(NDS32_SR_PFM_CTL); - val &= ~(PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); - val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void nds32_pmu_reset(void *info) -{ - u32 val = 0; - - val |= (PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); - __nds32__mtsr(val, NDS32_SR_PFM_CTL); - __nds32__mtsr(0, NDS32_SR_PFM_CTL); - __nds32__mtsr(0, NDS32_SR_PFMC0); - __nds32__mtsr(0, NDS32_SR_PFMC1); - __nds32__mtsr(0, NDS32_SR_PFMC2); -} - -static void nds32_pmu_init(struct nds32_pmu *cpu_pmu) -{ - cpu_pmu->handle_irq = nds32_pmu_handle_irq; - cpu_pmu->enable = nds32_pmu_enable_event; - cpu_pmu->disable = nds32_pmu_disable_event; - cpu_pmu->read_counter = nds32_pmu_read_counter; - cpu_pmu->write_counter = nds32_pmu_write_counter; - cpu_pmu->get_event_idx = nds32_pmu_get_event_idx; - cpu_pmu->start = nds32_pmu_start; - cpu_pmu->stop = nds32_pmu_stop; - cpu_pmu->reset = nds32_pmu_reset; - cpu_pmu->max_period = 0xFFFFFFFF; /* Maximum counts */ -}; - -static u32 nds32_read_num_pfm_events(void) -{ - /* NDS32 SPAv3 PMU support 3 counter */ - return 3; -} - -static int device_pmu_init(struct nds32_pmu *cpu_pmu) -{ - nds32_pmu_init(cpu_pmu); - /* - * This name should be devive-specific name, whatever you like :) - * I think "PMU" will be a good generic name. - */ - cpu_pmu->name = "nds32v3-pmu"; - cpu_pmu->map_event = nds32_spav3_map_event; - cpu_pmu->num_events = nds32_read_num_pfm_events(); - cpu_pmu->set_event_filter = nds32_pmu_set_event_filter; - return 0; -} - -/* - * CPU PMU identification and probing. - */ -static int probe_current_pmu(struct nds32_pmu *pmu) -{ - int ret; - - get_cpu(); - ret = -ENODEV; - /* - * If ther are various CPU types with its own PMU, initialize with - * - * the corresponding one - */ - device_pmu_init(pmu); - put_cpu(); - return ret; -} - -static void nds32_pmu_enable(struct pmu *pmu) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); - struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); - int enabled = bitmap_weight(hw_events->used_mask, - nds32_pmu->num_events); - - if (enabled) - nds32_pmu->start(nds32_pmu); -} - -static void nds32_pmu_disable(struct pmu *pmu) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); - - nds32_pmu->stop(nds32_pmu); -} - -static void nds32_pmu_release_hardware(struct nds32_pmu *nds32_pmu) -{ - nds32_pmu->free_irq(nds32_pmu); - pm_runtime_put_sync(&nds32_pmu->plat_device->dev); -} - -static irqreturn_t nds32_pmu_dispatch_irq(int irq, void *dev) -{ - struct nds32_pmu *nds32_pmu = (struct nds32_pmu *)dev; - int ret; - u64 start_clock, finish_clock; - - start_clock = local_clock(); - ret = nds32_pmu->handle_irq(irq, dev); - finish_clock = local_clock(); - - perf_sample_event_took(finish_clock - start_clock); - return ret; -} - -static int nds32_pmu_reserve_hardware(struct nds32_pmu *nds32_pmu) -{ - int err; - struct platform_device *pmu_device = nds32_pmu->plat_device; - - if (!pmu_device) - return -ENODEV; - - pm_runtime_get_sync(&pmu_device->dev); - err = nds32_pmu->request_irq(nds32_pmu, nds32_pmu_dispatch_irq); - if (err) { - nds32_pmu_release_hardware(nds32_pmu); - return err; - } - - return 0; -} - -static int -validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, - struct perf_event *event) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - - if (is_software_event(event)) - return 1; - - if (event->pmu != pmu) - return 0; - - if (event->state < PERF_EVENT_STATE_OFF) - return 1; - - if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) - return 1; - - return nds32_pmu->get_event_idx(hw_events, event) >= 0; -} - -static int validate_group(struct perf_event *event) -{ - struct perf_event *sibling, *leader = event->group_leader; - struct pmu_hw_events fake_pmu; - DECLARE_BITMAP(fake_used_mask, MAX_COUNTERS); - /* - * Initialize the fake PMU. We only need to populate the - * used_mask for the purposes of validation. - */ - memset(fake_used_mask, 0, sizeof(fake_used_mask)); - - if (!validate_event(event->pmu, &fake_pmu, leader)) - return -EINVAL; - - for_each_sibling_event(sibling, leader) { - if (!validate_event(event->pmu, &fake_pmu, sibling)) - return -EINVAL; - } - - if (!validate_event(event->pmu, &fake_pmu, event)) - return -EINVAL; - - return 0; -} - -static int __hw_perf_event_init(struct perf_event *event) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int mapping; - - mapping = nds32_pmu->map_event(event); - - if (mapping < 0) { - pr_debug("event %x:%llx not supported\n", event->attr.type, - event->attr.config); - return mapping; - } - - /* - * We don't assign an index until we actually place the event onto - * hardware. Use -1 to signify that we haven't decided where to put it - * yet. For SMP systems, each core has it's own PMU so we can't do any - * clever allocation or constraints checking at this point. - */ - hwc->idx = -1; - hwc->config_base = 0; - hwc->config = 0; - hwc->event_base = 0; - - /* - * Check whether we need to exclude the counter from certain modes. - */ - if ((!nds32_pmu->set_event_filter || - nds32_pmu->set_event_filter(hwc, &event->attr)) && - event_requires_mode_exclusion(&event->attr)) { - pr_debug - ("NDS performance counters do not support mode exclusion\n"); - return -EOPNOTSUPP; - } - - /* - * Store the event encoding into the config_base field. - */ - hwc->config_base |= (unsigned long)mapping; - - if (!hwc->sample_period) { - /* - * For non-sampling runs, limit the sample_period to half - * of the counter width. That way, the new counter value - * is far less likely to overtake the previous one unless - * you have some serious IRQ latency issues. - */ - hwc->sample_period = nds32_pmu->max_period >> 1; - hwc->last_period = hwc->sample_period; - local64_set(&hwc->period_left, hwc->sample_period); - } - - if (event->group_leader != event) { - if (validate_group(event) != 0) - return -EINVAL; - } - - return 0; -} - -static int nds32_pmu_event_init(struct perf_event *event) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - int err = 0; - atomic_t *active_events = &nds32_pmu->active_events; - - /* does not support taken branch sampling */ - if (has_branch_stack(event)) - return -EOPNOTSUPP; - - if (nds32_pmu->map_event(event) == -ENOENT) - return -ENOENT; - - if (!atomic_inc_not_zero(active_events)) { - if (atomic_read(active_events) == 0) { - /* Register irq handler */ - err = nds32_pmu_reserve_hardware(nds32_pmu); - } - - if (!err) - atomic_inc(active_events); - } - - if (err) - return err; - - err = __hw_perf_event_init(event); - - return err; -} - -static void nds32_start(struct perf_event *event, int flags) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - /* - * NDS pmu always has to reprogram the period, so ignore - * PERF_EF_RELOAD, see the comment below. - */ - if (flags & PERF_EF_RELOAD) - WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); - - hwc->state = 0; - /* Set the period for the event. */ - nds32_pmu_event_set_period(event); - - nds32_pmu->enable(event); -} - -static int nds32_pmu_add(struct perf_event *event, int flags) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); - struct hw_perf_event *hwc = &event->hw; - int idx; - int err = 0; - - perf_pmu_disable(event->pmu); - - /* If we don't have a space for the counter then finish early. */ - idx = nds32_pmu->get_event_idx(hw_events, event); - if (idx < 0) { - err = idx; - goto out; - } - - /* - * If there is an event in the counter we are going to use then make - * sure it is disabled. - */ - event->hw.idx = idx; - nds32_pmu->disable(event); - hw_events->events[idx] = event; - - hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; - if (flags & PERF_EF_START) - nds32_start(event, PERF_EF_RELOAD); - - /* Propagate our changes to the userspace mapping. */ - perf_event_update_userpage(event); - -out: - perf_pmu_enable(event->pmu); - return err; -} - -u64 nds32_pmu_event_update(struct perf_event *event) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - u64 delta, prev_raw_count, new_raw_count; - -again: - prev_raw_count = local64_read(&hwc->prev_count); - new_raw_count = nds32_pmu->read_counter(event); - - if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, - new_raw_count) != prev_raw_count) { - goto again; - } - /* - * Whether overflow or not, "unsigned substraction" - * will always get their delta - */ - delta = (new_raw_count - prev_raw_count) & nds32_pmu->max_period; - - local64_add(delta, &event->count); - local64_sub(delta, &hwc->period_left); - - return new_raw_count; -} - -static void nds32_stop(struct perf_event *event, int flags) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - /* - * NDS pmu always has to update the counter, so ignore - * PERF_EF_UPDATE, see comments in nds32_start(). - */ - if (!(hwc->state & PERF_HES_STOPPED)) { - nds32_pmu->disable(event); - nds32_pmu_event_update(event); - hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; - } -} - -static void nds32_pmu_del(struct perf_event *event, int flags) -{ - struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); - struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - nds32_stop(event, PERF_EF_UPDATE); - hw_events->events[idx] = NULL; - clear_bit(idx, hw_events->used_mask); - - perf_event_update_userpage(event); -} - -static void nds32_pmu_read(struct perf_event *event) -{ - nds32_pmu_event_update(event); -} - -/* Please refer to SPAv3 for more hardware specific details */ -PMU_FORMAT_ATTR(event, "config:0-63"); - -static struct attribute *nds32_arch_formats_attr[] = { - &format_attr_event.attr, - NULL, -}; - -static struct attribute_group nds32_pmu_format_group = { - .name = "format", - .attrs = nds32_arch_formats_attr, -}; - -static ssize_t nds32_pmu_cpumask_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - return 0; -} - -static DEVICE_ATTR(cpus, 0444, nds32_pmu_cpumask_show, NULL); - -static struct attribute *nds32_pmu_common_attrs[] = { - &dev_attr_cpus.attr, - NULL, -}; - -static struct attribute_group nds32_pmu_common_group = { - .attrs = nds32_pmu_common_attrs, -}; - -static const struct attribute_group *nds32_pmu_attr_groups[] = { - &nds32_pmu_format_group, - &nds32_pmu_common_group, - NULL, -}; - -static void nds32_init(struct nds32_pmu *nds32_pmu) -{ - atomic_set(&nds32_pmu->active_events, 0); - - nds32_pmu->pmu = (struct pmu) { - .pmu_enable = nds32_pmu_enable, - .pmu_disable = nds32_pmu_disable, - .attr_groups = nds32_pmu_attr_groups, - .event_init = nds32_pmu_event_init, - .add = nds32_pmu_add, - .del = nds32_pmu_del, - .start = nds32_start, - .stop = nds32_stop, - .read = nds32_pmu_read, - }; -} - -int nds32_pmu_register(struct nds32_pmu *nds32_pmu, int type) -{ - nds32_init(nds32_pmu); - pm_runtime_enable(&nds32_pmu->plat_device->dev); - pr_info("enabled with %s PMU driver, %d counters available\n", - nds32_pmu->name, nds32_pmu->num_events); - return perf_pmu_register(&nds32_pmu->pmu, nds32_pmu->name, type); -} - -static struct pmu_hw_events *cpu_pmu_get_cpu_events(void) -{ - return this_cpu_ptr(&cpu_hw_events); -} - -static int cpu_pmu_request_irq(struct nds32_pmu *cpu_pmu, irq_handler_t handler) -{ - int err, irq, irqs; - struct platform_device *pmu_device = cpu_pmu->plat_device; - - if (!pmu_device) - return -ENODEV; - - irqs = min(pmu_device->num_resources, num_possible_cpus()); - if (irqs < 1) { - pr_err("no irqs for PMUs defined\n"); - return -ENODEV; - } - - irq = platform_get_irq(pmu_device, 0); - err = request_irq(irq, handler, IRQF_NOBALANCING, "nds32-pfm", - cpu_pmu); - if (err) { - pr_err("unable to request IRQ%d for NDS PMU counters\n", - irq); - return err; - } - return 0; -} - -static void cpu_pmu_free_irq(struct nds32_pmu *cpu_pmu) -{ - int irq; - struct platform_device *pmu_device = cpu_pmu->plat_device; - - irq = platform_get_irq(pmu_device, 0); - if (irq >= 0) - free_irq(irq, cpu_pmu); -} - -static void cpu_pmu_init(struct nds32_pmu *cpu_pmu) -{ - int cpu; - struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu); - - raw_spin_lock_init(&events->pmu_lock); - - cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events; - cpu_pmu->request_irq = cpu_pmu_request_irq; - cpu_pmu->free_irq = cpu_pmu_free_irq; - - /* Ensure the PMU has sane values out of reset. */ - if (cpu_pmu->reset) - on_each_cpu(cpu_pmu->reset, cpu_pmu, 1); -} - -static const struct of_device_id cpu_pmu_of_device_ids[] = { - {.compatible = "andestech,nds32v3-pmu", - .data = device_pmu_init}, - {}, -}; - -static int cpu_pmu_device_probe(struct platform_device *pdev) -{ - const struct of_device_id *of_id; - int (*init_fn)(struct nds32_pmu *nds32_pmu); - struct device_node *node = pdev->dev.of_node; - struct nds32_pmu *pmu; - int ret = -ENODEV; - - if (cpu_pmu) { - pr_notice("[perf] attempt to register multiple PMU devices!\n"); - return -ENOSPC; - } - - pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); - if (!pmu) - return -ENOMEM; - - of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node); - if (node && of_id) { - init_fn = of_id->data; - ret = init_fn(pmu); - } else { - ret = probe_current_pmu(pmu); - } - - if (ret) { - pr_notice("[perf] failed to probe PMU!\n"); - goto out_free; - } - - cpu_pmu = pmu; - cpu_pmu->plat_device = pdev; - cpu_pmu_init(cpu_pmu); - ret = nds32_pmu_register(cpu_pmu, PERF_TYPE_RAW); - - if (!ret) - return 0; - -out_free: - pr_notice("[perf] failed to register PMU devices!\n"); - kfree(pmu); - return ret; -} - -static struct platform_driver cpu_pmu_driver = { - .driver = { - .name = "nds32-pfm", - .of_match_table = cpu_pmu_of_device_ids, - }, - .probe = cpu_pmu_device_probe, - .id_table = cpu_pmu_plat_device_ids, -}; - -static int __init register_pmu_driver(void) -{ - int err = 0; - - err = platform_driver_register(&cpu_pmu_driver); - if (err) - pr_notice("[perf] PMU initialization failed\n"); - else - pr_notice("[perf] PMU initialization done\n"); - - return err; -} - -device_initcall(register_pmu_driver); - -/* - * References: arch/nds32/kernel/traps.c:__dump() - * You will need to know the NDS ABI first. - */ -static int unwind_frame_kernel(struct stackframe *frame) -{ - int graph = 0; -#ifdef CONFIG_FRAME_POINTER - /* 0x3 means misalignment */ - if (!kstack_end((void *)frame->fp) && - !((unsigned long)frame->fp & 0x3) && - ((unsigned long)frame->fp >= TASK_SIZE)) { - /* - * The array index is based on the ABI, the below graph - * illustrate the reasons. - * Function call procedure: "smw" and "lmw" will always - * update SP and FP for you automatically. - * - * Stack Relative Address - * | | 0 - * ---- - * |LP| <-- SP(before smw) <-- FP(after smw) -1 - * ---- - * |FP| -2 - * ---- - * | | <-- SP(after smw) -3 - */ - frame->lp = ((unsigned long *)frame->fp)[-1]; - frame->fp = ((unsigned long *)frame->fp)[FP_OFFSET]; - /* make sure CONFIG_FUNCTION_GRAPH_TRACER is turned on */ - if (__kernel_text_address(frame->lp)) - frame->lp = ftrace_graph_ret_addr - (NULL, &graph, frame->lp, NULL); - - return 0; - } else { - return -EPERM; - } -#else - /* - * You can refer to arch/nds32/kernel/traps.c:__dump() - * Treat "sp" as "fp", but the "sp" is one frame ahead of "fp". - * And, the "sp" is not always correct. - * - * Stack Relative Address - * | | 0 - * ---- - * |LP| <-- SP(before smw) -1 - * ---- - * | | <-- SP(after smw) -2 - * ---- - */ - if (!kstack_end((void *)frame->sp)) { - frame->lp = ((unsigned long *)frame->sp)[1]; - /* TODO: How to deal with the value in first - * "sp" is not correct? - */ - if (__kernel_text_address(frame->lp)) - frame->lp = ftrace_graph_ret_addr - (tsk, &graph, frame->lp, NULL); - - frame->sp = ((unsigned long *)frame->sp) + 1; - - return 0; - } else { - return -EPERM; - } -#endif -} - -static void notrace -walk_stackframe(struct stackframe *frame, - int (*fn_record)(struct stackframe *, void *), - void *data) -{ - while (1) { - int ret; - - if (fn_record(frame, data)) - break; - - ret = unwind_frame_kernel(frame); - if (ret < 0) - break; - } -} - -/* - * Gets called by walk_stackframe() for every stackframe. This will be called - * whist unwinding the stackframe and is like a subroutine return so we use - * the PC. - */ -static int callchain_trace(struct stackframe *fr, void *data) -{ - struct perf_callchain_entry_ctx *entry = data; - - perf_callchain_store(entry, fr->lp); - return 0; -} - -/* - * Get the return address for a single stackframe and return a pointer to the - * next frame tail. - */ -static unsigned long -user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) -{ - struct frame_tail buftail; - unsigned long lp = 0; - unsigned long *user_frame_tail = - (unsigned long *)(fp - (unsigned long)sizeof(buftail)); - - /* Check accessibility of one struct frame_tail beyond */ - if (!access_ok(user_frame_tail, sizeof(buftail))) - return 0; - if (__copy_from_user_inatomic - (&buftail, user_frame_tail, sizeof(buftail))) - return 0; - - /* - * Refer to unwind_frame_kernel() for more illurstration - */ - lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ - fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ - perf_callchain_store(entry, lp); - return fp; -} - -static unsigned long -user_backtrace_opt_size(struct perf_callchain_entry_ctx *entry, - unsigned long fp) -{ - struct frame_tail_opt_size buftail; - unsigned long lp = 0; - - unsigned long *user_frame_tail = - (unsigned long *)(fp - (unsigned long)sizeof(buftail)); - - /* Check accessibility of one struct frame_tail beyond */ - if (!access_ok(user_frame_tail, sizeof(buftail))) - return 0; - if (__copy_from_user_inatomic - (&buftail, user_frame_tail, sizeof(buftail))) - return 0; - - /* - * Refer to unwind_frame_kernel() for more illurstration - */ - lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ - fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ - - perf_callchain_store(entry, lp); - return fp; -} - -/* - * This will be called when the target is in user mode - * This function will only be called when we use - * "PERF_SAMPLE_CALLCHAIN" in - * kernel/events/core.c:perf_prepare_sample() - * - * How to trigger perf_callchain_[user/kernel] : - * $ perf record -e cpu-clock --call-graph fp ./program - * $ perf report --call-graph - */ -unsigned long leaf_fp; -void -perf_callchain_user(struct perf_callchain_entry_ctx *entry, - struct pt_regs *regs) -{ - unsigned long fp = 0; - unsigned long gp = 0; - unsigned long lp = 0; - unsigned long sp = 0; - unsigned long *user_frame_tail; - - leaf_fp = 0; - - perf_callchain_store(entry, regs->ipc); - fp = regs->fp; - gp = regs->gp; - lp = regs->lp; - sp = regs->sp; - if (entry->nr < PERF_MAX_STACK_DEPTH && - (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { - user_frame_tail = - (unsigned long *)(fp - (unsigned long)sizeof(fp)); - - if (!access_ok(user_frame_tail, sizeof(fp))) - return; - - if (__copy_from_user_inatomic - (&leaf_fp, user_frame_tail, sizeof(fp))) - return; - - if (leaf_fp == lp) { - /* - * Maybe this is non leaf function - * with optimize for size, - * or maybe this is the function - * with optimize for size - */ - struct frame_tail buftail; - - user_frame_tail = - (unsigned long *)(fp - - (unsigned long)sizeof(buftail)); - - if (!access_ok(user_frame_tail, sizeof(buftail))) - return; - - if (__copy_from_user_inatomic - (&buftail, user_frame_tail, sizeof(buftail))) - return; - - if (buftail.stack_fp == gp) { - /* non leaf function with optimize - * for size condition - */ - struct frame_tail_opt_size buftail_opt_size; - - user_frame_tail = - (unsigned long *)(fp - (unsigned long) - sizeof(buftail_opt_size)); - - if (!access_ok(user_frame_tail, - sizeof(buftail_opt_size))) - return; - - if (__copy_from_user_inatomic - (&buftail_opt_size, user_frame_tail, - sizeof(buftail_opt_size))) - return; - - perf_callchain_store(entry, lp); - fp = buftail_opt_size.stack_fp; - - while ((entry->nr < PERF_MAX_STACK_DEPTH) && - (unsigned long)fp && - !((unsigned long)fp & 0x7) && - fp > sp) { - sp = fp; - fp = user_backtrace_opt_size(entry, fp); - } - - } else { - /* this is the function - * without optimize for size - */ - fp = buftail.stack_fp; - perf_callchain_store(entry, lp); - while ((entry->nr < PERF_MAX_STACK_DEPTH) && - (unsigned long)fp && - !((unsigned long)fp & 0x7) && - fp > sp) { - sp = fp; - fp = user_backtrace(entry, fp); - } - } - } else { - /* this is leaf function */ - fp = leaf_fp; - perf_callchain_store(entry, lp); - - /* previous function callcahin */ - while ((entry->nr < PERF_MAX_STACK_DEPTH) && - (unsigned long)fp && - !((unsigned long)fp & 0x7) && fp > sp) { - sp = fp; - fp = user_backtrace(entry, fp); - } - } - return; - } -} - -/* This will be called when the target is in kernel mode */ -void -perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, - struct pt_regs *regs) -{ - struct stackframe fr; - - fr.fp = regs->fp; - fr.lp = regs->lp; - fr.sp = regs->sp; - walk_stackframe(&fr, callchain_trace, entry); -} - -unsigned long perf_instruction_pointer(struct pt_regs *regs) -{ - return instruction_pointer(regs); -} - -unsigned long perf_misc_flags(struct pt_regs *regs) -{ - int misc = 0; - - if (user_mode(regs)) - misc |= PERF_RECORD_MISC_USER; - else - misc |= PERF_RECORD_MISC_KERNEL; - - return misc; -} diff --git a/arch/nds32/kernel/pm.c b/arch/nds32/kernel/pm.c deleted file mode 100644 index e25700e125d8..000000000000 --- a/arch/nds32/kernel/pm.c +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2008-2017 Andes Technology Corporation - -#include <linux/init.h> -#include <linux/suspend.h> -#include <linux/device.h> -#include <linux/printk.h> -#include <asm/suspend.h> -#include <nds32_intrinsic.h> - -unsigned int resume_addr; -unsigned int *phy_addr_sp_tmp; - -static void nds32_suspend2ram(void) -{ - pgd_t *pgdv; - p4d_t *p4dv; - pud_t *pudv; - pmd_t *pmdv; - pte_t *ptev; - - pgdv = (pgd_t *)__va((__nds32__mfsr(NDS32_SR_L1_PPTB) & - L1_PPTB_mskBASE)) + pgd_index((unsigned int)cpu_resume); - - p4dv = p4d_offset(pgdv, (unsigned int)cpu_resume); - pudv = pud_offset(p4dv, (unsigned int)cpu_resume); - pmdv = pmd_offset(pudv, (unsigned int)cpu_resume); - ptev = pte_offset_map(pmdv, (unsigned int)cpu_resume); - - resume_addr = ((*ptev) & TLB_DATA_mskPPN) - | ((unsigned int)cpu_resume & 0x00000fff); - - suspend2ram(); -} - -static void nds32_suspend_cpu(void) -{ - while (!(__nds32__mfsr(NDS32_SR_INT_PEND) & wake_mask)) - __asm__ volatile ("standby no_wake_grant\n\t"); -} - -static int nds32_pm_valid(suspend_state_t state) -{ - switch (state) { - case PM_SUSPEND_ON: - case PM_SUSPEND_STANDBY: - case PM_SUSPEND_MEM: - return 1; - default: - return 0; - } -} - -static int nds32_pm_enter(suspend_state_t state) -{ - pr_debug("%s:state:%d\n", __func__, state); - switch (state) { - case PM_SUSPEND_STANDBY: - nds32_suspend_cpu(); - return 0; - case PM_SUSPEND_MEM: - nds32_suspend2ram(); - return 0; - default: - return -EINVAL; - } -} - -static const struct platform_suspend_ops nds32_pm_ops = { - .valid = nds32_pm_valid, - .enter = nds32_pm_enter, -}; - -static int __init nds32_pm_init(void) -{ - pr_debug("Enter %s\n", __func__); - suspend_set_ops(&nds32_pm_ops); - return 0; -} -late_initcall(nds32_pm_init); diff --git a/arch/nds32/kernel/process.c b/arch/nds32/kernel/process.c deleted file mode 100644 index 49fab9e39cbf..000000000000 --- a/arch/nds32/kernel/process.c +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/sched.h> -#include <linux/sched/debug.h> -#include <linux/sched/task_stack.h> -#include <linux/delay.h> -#include <linux/kallsyms.h> -#include <linux/uaccess.h> -#include <asm/elf.h> -#include <asm/proc-fns.h> -#include <asm/fpu.h> -#include <linux/ptrace.h> -#include <linux/reboot.h> - -#if IS_ENABLED(CONFIG_LAZY_FPU) -struct task_struct *last_task_used_math; -#endif - -extern void setup_mm_for_reboot(char mode); - -extern inline void arch_reset(char mode) -{ - if (mode == 's') { - /* Use cpu handler, jump to 0 */ - cpu_reset(0); - } -} - -void (*pm_power_off) (void); -EXPORT_SYMBOL(pm_power_off); - -static char reboot_mode_nds32 = 'h'; - -int __init reboot_setup(char *str) -{ - reboot_mode_nds32 = str[0]; - return 1; -} - -static int cpub_pwroff(void) -{ - return 0; -} - -__setup("reboot=", reboot_setup); - -void machine_halt(void) -{ - cpub_pwroff(); -} - -EXPORT_SYMBOL(machine_halt); - -void machine_power_off(void) -{ - if (pm_power_off) - pm_power_off(); -} - -EXPORT_SYMBOL(machine_power_off); - -void machine_restart(char *cmd) -{ - /* - * Clean and disable cache, and turn off interrupts - */ - cpu_proc_fin(); - - /* - * Tell the mm system that we are going to reboot - - * we may need it to insert some 1:1 mappings so that - * soft boot works. - */ - setup_mm_for_reboot(reboot_mode_nds32); - - /* Execute kernel restart handler call chain */ - do_kernel_restart(cmd); - - /* - * Now call the architecture specific reboot code. - */ - arch_reset(reboot_mode_nds32); - - /* - * Whoops - the architecture was unable to reboot. - * Tell the user! - */ - mdelay(1000); - pr_info("Reboot failed -- System halted\n"); - while (1) ; -} - -EXPORT_SYMBOL(machine_restart); - -void show_regs(struct pt_regs *regs) -{ - printk("PC is at %pS\n", (void *)instruction_pointer(regs)); - printk("LP is at %pS\n", (void *)regs->lp); - pr_info("pc : [<%08lx>] lp : [<%08lx>] %s\n" - "sp : %08lx fp : %08lx gp : %08lx\n", - instruction_pointer(regs), - regs->lp, print_tainted(), regs->sp, regs->fp, regs->gp); - pr_info("r25: %08lx r24: %08lx\n", regs->uregs[25], regs->uregs[24]); - - pr_info("r23: %08lx r22: %08lx r21: %08lx r20: %08lx\n", - regs->uregs[23], regs->uregs[22], - regs->uregs[21], regs->uregs[20]); - pr_info("r19: %08lx r18: %08lx r17: %08lx r16: %08lx\n", - regs->uregs[19], regs->uregs[18], - regs->uregs[17], regs->uregs[16]); - pr_info("r15: %08lx r14: %08lx r13: %08lx r12: %08lx\n", - regs->uregs[15], regs->uregs[14], - regs->uregs[13], regs->uregs[12]); - pr_info("r11: %08lx r10: %08lx r9 : %08lx r8 : %08lx\n", - regs->uregs[11], regs->uregs[10], - regs->uregs[9], regs->uregs[8]); - pr_info("r7 : %08lx r6 : %08lx r5 : %08lx r4 : %08lx\n", - regs->uregs[7], regs->uregs[6], regs->uregs[5], regs->uregs[4]); - pr_info("r3 : %08lx r2 : %08lx r1 : %08lx r0 : %08lx\n", - regs->uregs[3], regs->uregs[2], regs->uregs[1], regs->uregs[0]); - pr_info(" IRQs o%s Segment %s\n", - interrupts_enabled(regs) ? "n" : "ff", - uaccess_kernel() ? "kernel" : "user"); -} - -EXPORT_SYMBOL(show_regs); - -void exit_thread(struct task_struct *tsk) -{ -#if defined(CONFIG_FPU) && defined(CONFIG_LAZY_FPU) - if (last_task_used_math == tsk) - last_task_used_math = NULL; -#endif -} - -void flush_thread(void) -{ -#if defined(CONFIG_FPU) - clear_fpu(task_pt_regs(current)); - clear_used_math(); -# ifdef CONFIG_LAZY_FPU - if (last_task_used_math == current) - last_task_used_math = NULL; -# endif -#endif -} - -DEFINE_PER_CPU(struct task_struct *, __entry_task); - -asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); -int copy_thread(unsigned long clone_flags, unsigned long stack_start, - unsigned long stk_sz, struct task_struct *p, unsigned long tls) -{ - struct pt_regs *childregs = task_pt_regs(p); - - memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); - - if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - /* kernel thread fn */ - p->thread.cpu_context.r6 = stack_start; - /* kernel thread argument */ - p->thread.cpu_context.r7 = stk_sz; - } else { - *childregs = *current_pt_regs(); - if (stack_start) - childregs->sp = stack_start; - /* child get zero as ret. */ - childregs->uregs[0] = 0; - childregs->osp = 0; - if (clone_flags & CLONE_SETTLS) - childregs->uregs[25] = tls; - } - /* cpu context switching */ - p->thread.cpu_context.pc = (unsigned long)ret_from_fork; - p->thread.cpu_context.sp = (unsigned long)childregs; - -#if IS_ENABLED(CONFIG_FPU) - if (used_math()) { -# if !IS_ENABLED(CONFIG_LAZY_FPU) - unlazy_fpu(current); -# else - preempt_disable(); - if (last_task_used_math == current) - save_fpu(current); - preempt_enable(); -# endif - p->thread.fpu = current->thread.fpu; - clear_fpu(task_pt_regs(p)); - set_stopped_child_used_math(p); - } -#endif - -#ifdef CONFIG_HWZOL - childregs->lb = 0; - childregs->le = 0; - childregs->lc = 0; -#endif - - return 0; -} - -#if IS_ENABLED(CONFIG_FPU) -struct task_struct *_switch_fpu(struct task_struct *prev, struct task_struct *next) -{ -#if !IS_ENABLED(CONFIG_LAZY_FPU) - unlazy_fpu(prev); -#endif - if (!(next->flags & PF_KTHREAD)) - clear_fpu(task_pt_regs(next)); - return prev; -} -#endif - -/* - * fill in the fpe structure for a core dump... - */ -int dump_fpu(struct pt_regs *regs, elf_fpregset_t * fpu) -{ - int fpvalid = 0; -#if IS_ENABLED(CONFIG_FPU) - struct task_struct *tsk = current; - - fpvalid = tsk_used_math(tsk); - if (fpvalid) { - lose_fpu(); - memcpy(fpu, &tsk->thread.fpu, sizeof(*fpu)); - } -#endif - return fpvalid; -} - -EXPORT_SYMBOL(dump_fpu); - -unsigned long __get_wchan(struct task_struct *p) -{ - unsigned long fp, lr; - unsigned long stack_start, stack_end; - int count = 0; - - if (IS_ENABLED(CONFIG_FRAME_POINTER)) { - stack_start = (unsigned long)end_of_stack(p); - stack_end = (unsigned long)task_stack_page(p) + THREAD_SIZE; - - fp = thread_saved_fp(p); - do { - if (fp < stack_start || fp > stack_end) - return 0; - lr = ((unsigned long *)fp)[0]; - if (!in_sched_functions(lr)) - return lr; - fp = *(unsigned long *)(fp + 4); - } while (count++ < 16); - } - return 0; -} diff --git a/arch/nds32/kernel/ptrace.c b/arch/nds32/kernel/ptrace.c deleted file mode 100644 index d0eda870fbc2..000000000000 --- a/arch/nds32/kernel/ptrace.c +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/ptrace.h> -#include <linux/regset.h> -#include <linux/tracehook.h> -#include <linux/elf.h> -#include <linux/sched/task_stack.h> - -enum nds32_regset { - REGSET_GPR, -}; - -static int gpr_get(struct task_struct *target, - const struct user_regset *regset, - struct membuf to) -{ - return membuf_write(&to, &task_pt_regs(target)->user_regs, - sizeof(struct user_pt_regs)); -} - -static int gpr_set(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - const void *kbuf, const void __user * ubuf) -{ - int err; - struct user_pt_regs newregs = task_pt_regs(target)->user_regs; - - err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1); - if (err) - return err; - - task_pt_regs(target)->user_regs = newregs; - return 0; -} - -static const struct user_regset nds32_regsets[] = { - [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, - .n = sizeof(struct user_pt_regs) / sizeof(u32), - .size = sizeof(elf_greg_t), - .align = sizeof(elf_greg_t), - .regset_get = gpr_get, - .set = gpr_set} -}; - -static const struct user_regset_view nds32_user_view = { - .name = "nds32", - .e_machine = EM_NDS32, - .regsets = nds32_regsets, - .n = ARRAY_SIZE(nds32_regsets) -}; - -const struct user_regset_view *task_user_regset_view(struct task_struct *task) -{ - return &nds32_user_view; -} - -void ptrace_disable(struct task_struct *child) -{ - user_disable_single_step(child); -} - -/* do_ptrace() - * - * Provide ptrace defined service. - */ -long arch_ptrace(struct task_struct *child, long request, unsigned long addr, - unsigned long data) -{ - int ret = -EIO; - - switch (request) { - default: - ret = ptrace_request(child, request, addr, data); - break; - } - - return ret; -} - -void user_enable_single_step(struct task_struct *child) -{ - struct pt_regs *regs; - regs = task_pt_regs(child); - regs->ipsw |= PSW_mskHSS; - set_tsk_thread_flag(child, TIF_SINGLESTEP); -} - -void user_disable_single_step(struct task_struct *child) -{ - struct pt_regs *regs; - regs = task_pt_regs(child); - regs->ipsw &= ~PSW_mskHSS; - clear_tsk_thread_flag(child, TIF_SINGLESTEP); -} - -/* sys_trace() - * - * syscall trace handler. - */ - -asmlinkage int syscall_trace_enter(struct pt_regs *regs) -{ - if (test_thread_flag(TIF_SYSCALL_TRACE)) { - if (tracehook_report_syscall_entry(regs)) - forget_syscall(regs); - } - return regs->syscallno; -} - -asmlinkage void syscall_trace_leave(struct pt_regs *regs) -{ - int step = test_thread_flag(TIF_SINGLESTEP); - if (step || test_thread_flag(TIF_SYSCALL_TRACE)) - tracehook_report_syscall_exit(regs, step); - -} diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c deleted file mode 100644 index b3d34d646652..000000000000 --- a/arch/nds32/kernel/setup.c +++ /dev/null @@ -1,369 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/cpu.h> -#include <linux/memblock.h> -#include <linux/seq_file.h> -#include <linux/console.h> -#include <linux/screen_info.h> -#include <linux/delay.h> -#include <linux/dma-mapping.h> -#include <linux/of_fdt.h> -#include <linux/of_platform.h> -#include <asm/setup.h> -#include <asm/sections.h> -#include <asm/proc-fns.h> -#include <asm/cache_info.h> -#include <asm/elf.h> -#include <asm/fpu.h> -#include <nds32_intrinsic.h> - -#define HWCAP_MFUSR_PC 0x000001 -#define HWCAP_EXT 0x000002 -#define HWCAP_EXT2 0x000004 -#define HWCAP_FPU 0x000008 -#define HWCAP_AUDIO 0x000010 -#define HWCAP_BASE16 0x000020 -#define HWCAP_STRING 0x000040 -#define HWCAP_REDUCED_REGS 0x000080 -#define HWCAP_VIDEO 0x000100 -#define HWCAP_ENCRYPT 0x000200 -#define HWCAP_EDM 0x000400 -#define HWCAP_LMDMA 0x000800 -#define HWCAP_PFM 0x001000 -#define HWCAP_HSMP 0x002000 -#define HWCAP_TRACE 0x004000 -#define HWCAP_DIV 0x008000 -#define HWCAP_MAC 0x010000 -#define HWCAP_L2C 0x020000 -#define HWCAP_FPU_DP 0x040000 -#define HWCAP_V2 0x080000 -#define HWCAP_DX_REGS 0x100000 -#define HWCAP_HWPRE 0x200000 - -unsigned long cpu_id, cpu_rev, cpu_cfgid; -bool has_fpu = false; -char cpu_series; -char *endianness = NULL; - -unsigned int __atags_pointer __initdata; -unsigned int elf_hwcap; -EXPORT_SYMBOL(elf_hwcap); - -/* - * The following string table, must sync with HWCAP_xx bitmask, - * which is defined above - */ -static const char *hwcap_str[] = { - "mfusr_pc", - "perf1", - "perf2", - "fpu", - "audio", - "16b", - "string", - "reduced_regs", - "video", - "encrypt", - "edm", - "lmdma", - "pfm", - "hsmp", - "trace", - "div", - "mac", - "l2c", - "fpu_dp", - "v2", - "dx_regs", - "hw_pre", - NULL, -}; - -#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH -#define WRITE_METHOD "write through" -#else -#define WRITE_METHOD "write back" -#endif - -struct cache_info L1_cache_info[2]; -static void __init dump_cpu_info(int cpu) -{ - int i, p = 0; - char str[sizeof(hwcap_str) + 16]; - - for (i = 0; hwcap_str[i]; i++) { - if (elf_hwcap & (1 << i)) { - sprintf(str + p, "%s ", hwcap_str[i]); - p += strlen(hwcap_str[i]) + 1; - } - } - - pr_info("CPU%d Features: %s\n", cpu, str); - - L1_cache_info[ICACHE].ways = CACHE_WAY(ICACHE); - L1_cache_info[ICACHE].line_size = CACHE_LINE_SIZE(ICACHE); - L1_cache_info[ICACHE].sets = CACHE_SET(ICACHE); - L1_cache_info[ICACHE].size = - L1_cache_info[ICACHE].ways * L1_cache_info[ICACHE].line_size * - L1_cache_info[ICACHE].sets / 1024; - pr_info("L1I:%dKB/%dS/%dW/%dB\n", L1_cache_info[ICACHE].size, - L1_cache_info[ICACHE].sets, L1_cache_info[ICACHE].ways, - L1_cache_info[ICACHE].line_size); - L1_cache_info[DCACHE].ways = CACHE_WAY(DCACHE); - L1_cache_info[DCACHE].line_size = CACHE_LINE_SIZE(DCACHE); - L1_cache_info[DCACHE].sets = CACHE_SET(DCACHE); - L1_cache_info[DCACHE].size = - L1_cache_info[DCACHE].ways * L1_cache_info[DCACHE].line_size * - L1_cache_info[DCACHE].sets / 1024; - pr_info("L1D:%dKB/%dS/%dW/%dB\n", L1_cache_info[DCACHE].size, - L1_cache_info[DCACHE].sets, L1_cache_info[DCACHE].ways, - L1_cache_info[DCACHE].line_size); - pr_info("L1 D-Cache is %s\n", WRITE_METHOD); - if (L1_cache_info[DCACHE].size != L1_CACHE_BYTES) - pr_crit - ("The cache line size(%d) of this processor is not the same as L1_CACHE_BYTES(%d).\n", - L1_cache_info[DCACHE].size, L1_CACHE_BYTES); -#ifdef CONFIG_CPU_CACHE_ALIASING - { - int aliasing_num; - aliasing_num = - L1_cache_info[ICACHE].size * 1024 / PAGE_SIZE / - L1_cache_info[ICACHE].ways; - L1_cache_info[ICACHE].aliasing_num = aliasing_num; - L1_cache_info[ICACHE].aliasing_mask = - (aliasing_num - 1) << PAGE_SHIFT; - aliasing_num = - L1_cache_info[DCACHE].size * 1024 / PAGE_SIZE / - L1_cache_info[DCACHE].ways; - L1_cache_info[DCACHE].aliasing_num = aliasing_num; - L1_cache_info[DCACHE].aliasing_mask = - (aliasing_num - 1) << PAGE_SHIFT; - } -#endif -#ifdef CONFIG_FPU - /* Disable fpu and enable when it is used. */ - if (has_fpu) - disable_fpu(); -#endif -} - -static void __init setup_cpuinfo(void) -{ - unsigned long tmp = 0, cpu_name; - - cpu_dcache_inval_all(); - cpu_icache_inval_all(); - __nds32__isb(); - - cpu_id = (__nds32__mfsr(NDS32_SR_CPU_VER) & CPU_VER_mskCPUID) >> CPU_VER_offCPUID; - cpu_name = ((cpu_id) & 0xf0) >> 4; - cpu_series = cpu_name ? cpu_name - 10 + 'A' : 'N'; - cpu_id = cpu_id & 0xf; - cpu_rev = (__nds32__mfsr(NDS32_SR_CPU_VER) & CPU_VER_mskREV) >> CPU_VER_offREV; - cpu_cfgid = (__nds32__mfsr(NDS32_SR_CPU_VER) & CPU_VER_mskCFGID) >> CPU_VER_offCFGID; - - pr_info("CPU:%c%ld, CPU_VER 0x%08x(id %lu, rev %lu, cfg %lu)\n", - cpu_series, cpu_id, __nds32__mfsr(NDS32_SR_CPU_VER), cpu_id, cpu_rev, cpu_cfgid); - - elf_hwcap |= HWCAP_MFUSR_PC; - - if (((__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskBASEV) >> MSC_CFG_offBASEV) == 0) { - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskDIV) - elf_hwcap |= HWCAP_DIV; - - if ((__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskMAC) - || (cpu_id == 12 && cpu_rev < 4)) - elf_hwcap |= HWCAP_MAC; - } else { - elf_hwcap |= HWCAP_V2; - elf_hwcap |= HWCAP_DIV; - elf_hwcap |= HWCAP_MAC; - } - - if (cpu_cfgid & 0x0001) - elf_hwcap |= HWCAP_EXT; - - if (cpu_cfgid & 0x0002) - elf_hwcap |= HWCAP_BASE16; - - if (cpu_cfgid & 0x0004) - elf_hwcap |= HWCAP_EXT2; - - if (cpu_cfgid & 0x0008) { - elf_hwcap |= HWCAP_FPU; - has_fpu = true; - } - if (cpu_cfgid & 0x0010) - elf_hwcap |= HWCAP_STRING; - - if (__nds32__mfsr(NDS32_SR_MMU_CFG) & MMU_CFG_mskDE) - endianness = "MSB"; - else - endianness = "LSB"; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskEDM) - elf_hwcap |= HWCAP_EDM; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskLMDMA) - elf_hwcap |= HWCAP_LMDMA; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskPFM) - elf_hwcap |= HWCAP_PFM; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskHSMP) - elf_hwcap |= HWCAP_HSMP; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskTRACE) - elf_hwcap |= HWCAP_TRACE; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskAUDIO) - elf_hwcap |= HWCAP_AUDIO; - - if (__nds32__mfsr(NDS32_SR_MSC_CFG) & MSC_CFG_mskL2C) - elf_hwcap |= HWCAP_L2C; - -#ifdef CONFIG_HW_PRE - if (__nds32__mfsr(NDS32_SR_MISC_CTL) & MISC_CTL_makHWPRE_EN) - elf_hwcap |= HWCAP_HWPRE; -#endif - - tmp = __nds32__mfsr(NDS32_SR_CACHE_CTL); - if (!IS_ENABLED(CONFIG_CPU_DCACHE_DISABLE)) - tmp |= CACHE_CTL_mskDC_EN; - - if (!IS_ENABLED(CONFIG_CPU_ICACHE_DISABLE)) - tmp |= CACHE_CTL_mskIC_EN; - __nds32__mtsr_isb(tmp, NDS32_SR_CACHE_CTL); - - dump_cpu_info(smp_processor_id()); -} - -static void __init setup_memory(void) -{ - unsigned long ram_start_pfn; - unsigned long free_ram_start_pfn; - phys_addr_t memory_start, memory_end; - - memory_end = memory_start = 0; - - /* Find main memory where is the kernel */ - memory_start = memblock_start_of_DRAM(); - memory_end = memblock_end_of_DRAM(); - - if (!memory_end) { - panic("No memory!"); - } - - ram_start_pfn = PFN_UP(memblock_start_of_DRAM()); - /* free_ram_start_pfn is first page after kernel */ - free_ram_start_pfn = PFN_UP(__pa(&_end)); - max_pfn = PFN_DOWN(memblock_end_of_DRAM()); - /* it could update max_pfn */ - if (max_pfn - ram_start_pfn <= MAXMEM_PFN) - max_low_pfn = max_pfn; - else { - max_low_pfn = MAXMEM_PFN + ram_start_pfn; - if (!IS_ENABLED(CONFIG_HIGHMEM)) - max_pfn = MAXMEM_PFN + ram_start_pfn; - } - /* high_memory is related with VMALLOC */ - high_memory = (void *)__va(max_low_pfn * PAGE_SIZE); - min_low_pfn = free_ram_start_pfn; - - /* - * initialize the boot-time allocator (with low memory only). - * - * This makes the memory from the end of the kernel to the end of - * RAM usable. - */ - memblock_set_bottom_up(true); - memblock_reserve(PFN_PHYS(ram_start_pfn), PFN_PHYS(free_ram_start_pfn - ram_start_pfn)); - - early_init_fdt_reserve_self(); - early_init_fdt_scan_reserved_mem(); - - memblock_dump_all(); -} - -void __init setup_arch(char **cmdline_p) -{ - early_init_devtree(__atags_pointer ? \ - phys_to_virt(__atags_pointer) : __dtb_start); - - setup_cpuinfo(); - - setup_initial_init_mm(_stext, _etext, _edata, _end); - - /* setup bootmem allocator */ - setup_memory(); - - /* paging_init() sets up the MMU and marks all pages as reserved */ - paging_init(); - - /* invalidate all TLB entries because the new mapping is created */ - __nds32__tlbop_flua(); - - /* use generic way to parse */ - parse_early_param(); - - unflatten_and_copy_device_tree(); - - *cmdline_p = boot_command_line; - early_trap_init(); -} - -static int c_show(struct seq_file *m, void *v) -{ - int i; - - seq_printf(m, "Processor\t: %c%ld (id %lu, rev %lu, cfg %lu)\n", - cpu_series, cpu_id, cpu_id, cpu_rev, cpu_cfgid); - - seq_printf(m, "L1I\t\t: %luKB/%luS/%luW/%luB\n", - CACHE_SET(ICACHE) * CACHE_WAY(ICACHE) * - CACHE_LINE_SIZE(ICACHE) / 1024, CACHE_SET(ICACHE), - CACHE_WAY(ICACHE), CACHE_LINE_SIZE(ICACHE)); - - seq_printf(m, "L1D\t\t: %luKB/%luS/%luW/%luB\n", - CACHE_SET(DCACHE) * CACHE_WAY(DCACHE) * - CACHE_LINE_SIZE(DCACHE) / 1024, CACHE_SET(DCACHE), - CACHE_WAY(DCACHE), CACHE_LINE_SIZE(DCACHE)); - - seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", - loops_per_jiffy / (500000 / HZ), - (loops_per_jiffy / (5000 / HZ)) % 100); - - /* dump out the processor features */ - seq_puts(m, "Features\t: "); - - for (i = 0; hwcap_str[i]; i++) - if (elf_hwcap & (1 << i)) - seq_printf(m, "%s ", hwcap_str[i]); - - seq_puts(m, "\n\n"); - - return 0; -} - -static void *c_start(struct seq_file *m, loff_t * pos) -{ - return *pos < 1 ? (void *)1 : NULL; -} - -static void *c_next(struct seq_file *m, void *v, loff_t * pos) -{ - ++*pos; - return NULL; -} - -static void c_stop(struct seq_file *m, void *v) -{ -} - -struct seq_operations cpuinfo_op = { - .start = c_start, - .next = c_next, - .stop = c_stop, - .show = c_show -}; diff --git a/arch/nds32/kernel/signal.c b/arch/nds32/kernel/signal.c deleted file mode 100644 index 7e3ca430a223..000000000000 --- a/arch/nds32/kernel/signal.c +++ /dev/null @@ -1,384 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/errno.h> -#include <linux/signal.h> -#include <linux/ptrace.h> -#include <linux/personality.h> -#include <linux/freezer.h> -#include <linux/tracehook.h> -#include <linux/uaccess.h> - -#include <asm/cacheflush.h> -#include <asm/ucontext.h> -#include <asm/unistd.h> -#include <asm/fpu.h> - -#include <asm/ptrace.h> -#include <asm/vdso.h> - -struct rt_sigframe { - struct siginfo info; - struct ucontext uc; -}; -#if IS_ENABLED(CONFIG_FPU) -static inline int restore_sigcontext_fpu(struct pt_regs *regs, - struct sigcontext __user *sc) -{ - struct task_struct *tsk = current; - unsigned long used_math_flag; - int ret = 0; - - clear_used_math(); - __get_user_error(used_math_flag, &sc->used_math_flag, ret); - - if (!used_math_flag) - return 0; - set_used_math(); - -#if IS_ENABLED(CONFIG_LAZY_FPU) - preempt_disable(); - if (current == last_task_used_math) { - last_task_used_math = NULL; - disable_ptreg_fpu(regs); - } - preempt_enable(); -#else - clear_fpu(regs); -#endif - - return __copy_from_user(&tsk->thread.fpu, &sc->fpu, - sizeof(struct fpu_struct)); -} - -static inline int setup_sigcontext_fpu(struct pt_regs *regs, - struct sigcontext __user *sc) -{ - struct task_struct *tsk = current; - int ret = 0; - - __put_user_error(used_math(), &sc->used_math_flag, ret); - - if (!used_math()) - return ret; - - preempt_disable(); -#if IS_ENABLED(CONFIG_LAZY_FPU) - if (last_task_used_math == tsk) - save_fpu(last_task_used_math); -#else - unlazy_fpu(tsk); -#endif - ret = __copy_to_user(&sc->fpu, &tsk->thread.fpu, - sizeof(struct fpu_struct)); - preempt_enable(); - return ret; -} -#endif - -static int restore_sigframe(struct pt_regs *regs, - struct rt_sigframe __user * sf) -{ - sigset_t set; - int err; - - err = __copy_from_user(&set, &sf->uc.uc_sigmask, sizeof(set)); - if (err == 0) { - set_current_blocked(&set); - } - - __get_user_error(regs->uregs[0], &sf->uc.uc_mcontext.nds32_r0, err); - __get_user_error(regs->uregs[1], &sf->uc.uc_mcontext.nds32_r1, err); - __get_user_error(regs->uregs[2], &sf->uc.uc_mcontext.nds32_r2, err); - __get_user_error(regs->uregs[3], &sf->uc.uc_mcontext.nds32_r3, err); - __get_user_error(regs->uregs[4], &sf->uc.uc_mcontext.nds32_r4, err); - __get_user_error(regs->uregs[5], &sf->uc.uc_mcontext.nds32_r5, err); - __get_user_error(regs->uregs[6], &sf->uc.uc_mcontext.nds32_r6, err); - __get_user_error(regs->uregs[7], &sf->uc.uc_mcontext.nds32_r7, err); - __get_user_error(regs->uregs[8], &sf->uc.uc_mcontext.nds32_r8, err); - __get_user_error(regs->uregs[9], &sf->uc.uc_mcontext.nds32_r9, err); - __get_user_error(regs->uregs[10], &sf->uc.uc_mcontext.nds32_r10, err); - __get_user_error(regs->uregs[11], &sf->uc.uc_mcontext.nds32_r11, err); - __get_user_error(regs->uregs[12], &sf->uc.uc_mcontext.nds32_r12, err); - __get_user_error(regs->uregs[13], &sf->uc.uc_mcontext.nds32_r13, err); - __get_user_error(regs->uregs[14], &sf->uc.uc_mcontext.nds32_r14, err); - __get_user_error(regs->uregs[15], &sf->uc.uc_mcontext.nds32_r15, err); - __get_user_error(regs->uregs[16], &sf->uc.uc_mcontext.nds32_r16, err); - __get_user_error(regs->uregs[17], &sf->uc.uc_mcontext.nds32_r17, err); - __get_user_error(regs->uregs[18], &sf->uc.uc_mcontext.nds32_r18, err); - __get_user_error(regs->uregs[19], &sf->uc.uc_mcontext.nds32_r19, err); - __get_user_error(regs->uregs[20], &sf->uc.uc_mcontext.nds32_r20, err); - __get_user_error(regs->uregs[21], &sf->uc.uc_mcontext.nds32_r21, err); - __get_user_error(regs->uregs[22], &sf->uc.uc_mcontext.nds32_r22, err); - __get_user_error(regs->uregs[23], &sf->uc.uc_mcontext.nds32_r23, err); - __get_user_error(regs->uregs[24], &sf->uc.uc_mcontext.nds32_r24, err); - __get_user_error(regs->uregs[25], &sf->uc.uc_mcontext.nds32_r25, err); - - __get_user_error(regs->fp, &sf->uc.uc_mcontext.nds32_fp, err); - __get_user_error(regs->gp, &sf->uc.uc_mcontext.nds32_gp, err); - __get_user_error(regs->lp, &sf->uc.uc_mcontext.nds32_lp, err); - __get_user_error(regs->sp, &sf->uc.uc_mcontext.nds32_sp, err); - __get_user_error(regs->ipc, &sf->uc.uc_mcontext.nds32_ipc, err); -#if defined(CONFIG_HWZOL) - __get_user_error(regs->lc, &sf->uc.uc_mcontext.zol.nds32_lc, err); - __get_user_error(regs->le, &sf->uc.uc_mcontext.zol.nds32_le, err); - __get_user_error(regs->lb, &sf->uc.uc_mcontext.zol.nds32_lb, err); -#endif -#if IS_ENABLED(CONFIG_FPU) - err |= restore_sigcontext_fpu(regs, &sf->uc.uc_mcontext); -#endif - /* - * Avoid sys_rt_sigreturn() restarting. - */ - forget_syscall(regs); - return err; -} - -asmlinkage long sys_rt_sigreturn(struct pt_regs *regs) -{ - struct rt_sigframe __user *frame; - - /* Always make any pending restarted system calls return -EINTR */ - current->restart_block.fn = do_no_restart_syscall; - - /* - * Since we stacked the signal on a 64-bit boundary, - * then 'sp' should be two-word aligned here. If it's - * not, then the user is trying to mess with us. - */ - if (regs->sp & 7) - goto badframe; - - frame = (struct rt_sigframe __user *)regs->sp; - - if (!access_ok(frame, sizeof(*frame))) - goto badframe; - - if (restore_sigframe(regs, frame)) - goto badframe; - - if (restore_altstack(&frame->uc.uc_stack)) - goto badframe; - - return regs->uregs[0]; - -badframe: - force_sig(SIGSEGV); - return 0; -} - -static int -setup_sigframe(struct rt_sigframe __user * sf, struct pt_regs *regs, - sigset_t * set) -{ - int err = 0; - - __put_user_error(regs->uregs[0], &sf->uc.uc_mcontext.nds32_r0, err); - __put_user_error(regs->uregs[1], &sf->uc.uc_mcontext.nds32_r1, err); - __put_user_error(regs->uregs[2], &sf->uc.uc_mcontext.nds32_r2, err); - __put_user_error(regs->uregs[3], &sf->uc.uc_mcontext.nds32_r3, err); - __put_user_error(regs->uregs[4], &sf->uc.uc_mcontext.nds32_r4, err); - __put_user_error(regs->uregs[5], &sf->uc.uc_mcontext.nds32_r5, err); - __put_user_error(regs->uregs[6], &sf->uc.uc_mcontext.nds32_r6, err); - __put_user_error(regs->uregs[7], &sf->uc.uc_mcontext.nds32_r7, err); - __put_user_error(regs->uregs[8], &sf->uc.uc_mcontext.nds32_r8, err); - __put_user_error(regs->uregs[9], &sf->uc.uc_mcontext.nds32_r9, err); - __put_user_error(regs->uregs[10], &sf->uc.uc_mcontext.nds32_r10, err); - __put_user_error(regs->uregs[11], &sf->uc.uc_mcontext.nds32_r11, err); - __put_user_error(regs->uregs[12], &sf->uc.uc_mcontext.nds32_r12, err); - __put_user_error(regs->uregs[13], &sf->uc.uc_mcontext.nds32_r13, err); - __put_user_error(regs->uregs[14], &sf->uc.uc_mcontext.nds32_r14, err); - __put_user_error(regs->uregs[15], &sf->uc.uc_mcontext.nds32_r15, err); - __put_user_error(regs->uregs[16], &sf->uc.uc_mcontext.nds32_r16, err); - __put_user_error(regs->uregs[17], &sf->uc.uc_mcontext.nds32_r17, err); - __put_user_error(regs->uregs[18], &sf->uc.uc_mcontext.nds32_r18, err); - __put_user_error(regs->uregs[19], &sf->uc.uc_mcontext.nds32_r19, err); - __put_user_error(regs->uregs[20], &sf->uc.uc_mcontext.nds32_r20, err); - - __put_user_error(regs->uregs[21], &sf->uc.uc_mcontext.nds32_r21, err); - __put_user_error(regs->uregs[22], &sf->uc.uc_mcontext.nds32_r22, err); - __put_user_error(regs->uregs[23], &sf->uc.uc_mcontext.nds32_r23, err); - __put_user_error(regs->uregs[24], &sf->uc.uc_mcontext.nds32_r24, err); - __put_user_error(regs->uregs[25], &sf->uc.uc_mcontext.nds32_r25, err); - __put_user_error(regs->fp, &sf->uc.uc_mcontext.nds32_fp, err); - __put_user_error(regs->gp, &sf->uc.uc_mcontext.nds32_gp, err); - __put_user_error(regs->lp, &sf->uc.uc_mcontext.nds32_lp, err); - __put_user_error(regs->sp, &sf->uc.uc_mcontext.nds32_sp, err); - __put_user_error(regs->ipc, &sf->uc.uc_mcontext.nds32_ipc, err); -#if defined(CONFIG_HWZOL) - __put_user_error(regs->lc, &sf->uc.uc_mcontext.zol.nds32_lc, err); - __put_user_error(regs->le, &sf->uc.uc_mcontext.zol.nds32_le, err); - __put_user_error(regs->lb, &sf->uc.uc_mcontext.zol.nds32_lb, err); -#endif -#if IS_ENABLED(CONFIG_FPU) - err |= setup_sigcontext_fpu(regs, &sf->uc.uc_mcontext); -#endif - - __put_user_error(current->thread.trap_no, &sf->uc.uc_mcontext.trap_no, - err); - __put_user_error(current->thread.error_code, - &sf->uc.uc_mcontext.error_code, err); - __put_user_error(current->thread.address, - &sf->uc.uc_mcontext.fault_address, err); - __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); - - err |= __copy_to_user(&sf->uc.uc_sigmask, set, sizeof(*set)); - - return err; -} - -static inline void __user *get_sigframe(struct ksignal *ksig, - struct pt_regs *regs, int framesize) -{ - unsigned long sp; - - /* Default to using normal stack */ - sp = regs->sp; - - /* - * If we are on the alternate signal stack and would overflow it, don't. - * Return an always-bogus address instead so we will die with SIGSEGV. - */ - if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize))) - return (void __user __force *)(-1UL); - - /* This is the X/Open sanctioned signal stack switching. */ - sp = (sigsp(sp, ksig) - framesize); - - /* - * nds32 mandates 8-byte alignment - */ - sp &= ~0x7UL; - - return (void __user *)sp; -} - -static int -setup_return(struct pt_regs *regs, struct ksignal *ksig, void __user * frame) -{ - unsigned long handler = (unsigned long)ksig->ka.sa.sa_handler; - unsigned long retcode; - - retcode = VDSO_SYMBOL(current->mm->context.vdso, rt_sigtramp); - regs->uregs[0] = ksig->sig; - regs->sp = (unsigned long)frame; - regs->lp = retcode; - regs->ipc = handler; - - return 0; -} - -static int -setup_rt_frame(struct ksignal *ksig, sigset_t * set, struct pt_regs *regs) -{ - struct rt_sigframe __user *frame = - get_sigframe(ksig, regs, sizeof(*frame)); - int err = 0; - - if (!access_ok(frame, sizeof(*frame))) - return -EFAULT; - - __put_user_error(0, &frame->uc.uc_flags, err); - __put_user_error(NULL, &frame->uc.uc_link, err); - - err |= __save_altstack(&frame->uc.uc_stack, regs->sp); - err |= setup_sigframe(frame, regs, set); - if (err == 0) { - setup_return(regs, ksig, frame); - if (ksig->ka.sa.sa_flags & SA_SIGINFO) { - err |= copy_siginfo_to_user(&frame->info, &ksig->info); - regs->uregs[1] = (unsigned long)&frame->info; - regs->uregs[2] = (unsigned long)&frame->uc; - } - } - return err; -} - -/* - * OK, we're invoking a handler - */ -static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) -{ - int ret; - sigset_t *oldset = sigmask_to_save(); - - if (in_syscall(regs)) { - /* Avoid additional syscall restarting via ret_slow_syscall. */ - forget_syscall(regs); - - switch (regs->uregs[0]) { - case -ERESTART_RESTARTBLOCK: - case -ERESTARTNOHAND: - regs->uregs[0] = -EINTR; - break; - case -ERESTARTSYS: - if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { - regs->uregs[0] = -EINTR; - break; - } - fallthrough; - case -ERESTARTNOINTR: - regs->uregs[0] = regs->orig_r0; - regs->ipc -= 4; - break; - } - } - /* - * Set up the stack frame - */ - ret = setup_rt_frame(ksig, oldset, regs); - - signal_setup_done(ret, ksig, 0); -} - -/* - * Note that 'init' is a special process: it doesn't get signals it doesn't - * want to handle. Thus you cannot kill init even with a SIGKILL even by - * mistake. - * - * Note that we go through the signals twice: once to check the signals that - * the kernel can handle, and then we build all the user-level signal handling - * stack-frames in one go after that. - */ -static void do_signal(struct pt_regs *regs) -{ - struct ksignal ksig; - - if (get_signal(&ksig)) { - handle_signal(&ksig, regs); - return; - } - - /* - * If we were from a system call, check for system call restarting... - */ - if (in_syscall(regs)) { - /* Restart the system call - no handlers present */ - - /* Avoid additional syscall restarting via ret_slow_syscall. */ - forget_syscall(regs); - - switch (regs->uregs[0]) { - case -ERESTART_RESTARTBLOCK: - regs->uregs[15] = __NR_restart_syscall; - fallthrough; - case -ERESTARTNOHAND: - case -ERESTARTSYS: - case -ERESTARTNOINTR: - regs->uregs[0] = regs->orig_r0; - regs->ipc -= 0x4; - break; - } - } - restore_saved_sigmask(); -} - -asmlinkage void -do_notify_resume(struct pt_regs *regs, unsigned int thread_flags) -{ - if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_flags & _TIF_NOTIFY_RESUME) - tracehook_notify_resume(regs); -} diff --git a/arch/nds32/kernel/sleep.S b/arch/nds32/kernel/sleep.S deleted file mode 100644 index ca4e61f3656f..000000000000 --- a/arch/nds32/kernel/sleep.S +++ /dev/null @@ -1,131 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2017 Andes Technology Corporation */ - -#include <asm/memory.h> - -.data -.global sp_tmp -sp_tmp: -.long - -.text -.globl suspend2ram -.globl cpu_resume - -suspend2ram: - pushm $r0, $r31 -#if defined(CONFIG_HWZOL) - mfusr $r0, $lc - mfusr $r1, $le - mfusr $r2, $lb -#endif - mfsr $r3, $mr0 - mfsr $r4, $mr1 - mfsr $r5, $mr4 - mfsr $r6, $mr6 - mfsr $r7, $mr7 - mfsr $r8, $mr8 - mfsr $r9, $ir0 - mfsr $r10, $ir1 - mfsr $r11, $ir2 - mfsr $r12, $ir3 - mfsr $r13, $ir9 - mfsr $r14, $ir10 - mfsr $r15, $ir12 - mfsr $r16, $ir13 - mfsr $r17, $ir14 - mfsr $r18, $ir15 - pushm $r0, $r19 -#if defined(CONFIG_FPU) - jal store_fpu_for_suspend -#endif - tlbop FlushAll - isb - - // transfer $sp from va to pa - sethi $r0, hi20(PAGE_OFFSET) - ori $r0, $r0, lo12(PAGE_OFFSET) - movi $r2, PHYS_OFFSET - sub $r1, $sp, $r0 - add $r2, $r1, $r2 - - // store pa($sp) to sp_tmp - sethi $r1, hi20(sp_tmp) - swi $r2, [$r1 + lo12(sp_tmp)] - - pushm $r16, $r25 - pushm $r29, $r30 -#ifdef CONFIG_CACHE_L2 - jal dcache_wb_all_level -#else - jal cpu_dcache_wb_all -#endif - popm $r29, $r30 - popm $r16, $r25 - - // get wake_mask and loop in standby - la $r1, wake_mask - lwi $r1, [$r1] -self_loop: - standby wake_grant - mfsr $r2, $ir15 - and $r2, $r1, $r2 - beqz $r2, self_loop - - // set ipc to resume address - la $r1, resume_addr - lwi $r1, [$r1] - mtsr $r1, $ipc - isb - - // reset psw, turn off the address translation - li $r2, 0x7000a - mtsr $r2, $ipsw - isb - - iret -cpu_resume: - // translate the address of sp_tmp variable to pa - la $r1, sp_tmp - sethi $r0, hi20(PAGE_OFFSET) - ori $r0, $r0, lo12(PAGE_OFFSET) - movi $r2, PHYS_OFFSET - sub $r1, $r1, $r0 - add $r1, $r1, $r2 - - // access the sp_tmp to get stack pointer - lwi $sp, [$r1] - - popm $r0, $r19 -#if defined(CONFIG_HWZOL) - mtusr $r0, $lb - mtusr $r1, $lc - mtusr $r2, $le -#endif - mtsr $r3, $mr0 - mtsr $r4, $mr1 - mtsr $r5, $mr4 - mtsr $r6, $mr6 - mtsr $r7, $mr7 - mtsr $r8, $mr8 - // set original psw to ipsw - mtsr $r9, $ir1 - - mtsr $r11, $ir2 - mtsr $r12, $ir3 - - // set ipc to RR - la $r13, RR - mtsr $r13, $ir9 - - mtsr $r14, $ir10 - mtsr $r15, $ir12 - mtsr $r16, $ir13 - mtsr $r17, $ir14 - mtsr $r18, $ir15 - popm $r0, $r31 - - isb - iret -RR: - ret diff --git a/arch/nds32/kernel/stacktrace.c b/arch/nds32/kernel/stacktrace.c deleted file mode 100644 index d974c0c1c65f..000000000000 --- a/arch/nds32/kernel/stacktrace.c +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/sched/debug.h> -#include <linux/sched/task_stack.h> -#include <linux/stacktrace.h> -#include <linux/ftrace.h> - -void save_stack_trace(struct stack_trace *trace) -{ - save_stack_trace_tsk(current, trace); -} -EXPORT_SYMBOL_GPL(save_stack_trace); - -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) -{ - unsigned long *fpn; - int skip = trace->skip; - int savesched; - int graph_idx = 0; - - if (tsk == current) { - __asm__ __volatile__("\tori\t%0, $fp, #0\n":"=r"(fpn)); - savesched = 1; - } else { - fpn = (unsigned long *)thread_saved_fp(tsk); - savesched = 0; - } - - while (!kstack_end(fpn) && !((unsigned long)fpn & 0x3) - && (fpn >= (unsigned long *)TASK_SIZE)) { - unsigned long lpp, fpp; - - lpp = fpn[LP_OFFSET]; - fpp = fpn[FP_OFFSET]; - if (!__kernel_text_address(lpp)) - break; - else - lpp = ftrace_graph_ret_addr(tsk, &graph_idx, lpp, NULL); - - if (savesched || !in_sched_functions(lpp)) { - if (skip) { - skip--; - } else { - trace->entries[trace->nr_entries++] = lpp; - if (trace->nr_entries >= trace->max_entries) - break; - } - } - fpn = (unsigned long *)fpp; - } -} -EXPORT_SYMBOL_GPL(save_stack_trace_tsk); diff --git a/arch/nds32/kernel/sys_nds32.c b/arch/nds32/kernel/sys_nds32.c deleted file mode 100644 index cb2d1e219bb3..000000000000 --- a/arch/nds32/kernel/sys_nds32.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/syscalls.h> -#include <linux/uaccess.h> - -#include <asm/cachectl.h> -#include <asm/proc-fns.h> -#include <asm/fpu.h> -#include <asm/fp_udfiex_crtl.h> - -SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, - unsigned long, prot, unsigned long, flags, - unsigned long, fd, unsigned long, pgoff) -{ - if (pgoff & (~PAGE_MASK >> 12)) - return -EINVAL; - - return sys_mmap_pgoff(addr, len, prot, flags, fd, - pgoff >> (PAGE_SHIFT - 12)); -} - -SYSCALL_DEFINE4(fadvise64_64_wrapper,int, fd, int, advice, loff_t, offset, - loff_t, len) -{ - return sys_fadvise64_64(fd, offset, len, advice); -} - -SYSCALL_DEFINE3(cacheflush, unsigned int, start, unsigned int, end, int, cache) -{ - struct vm_area_struct *vma; - bool flushi = true, wbd = true; - - vma = find_vma(current->mm, start); - if (!vma) - return -EFAULT; - switch (cache) { - case ICACHE: - wbd = false; - break; - case DCACHE: - flushi = false; - break; - case BCACHE: - break; - default: - return -EINVAL; - } - cpu_cache_wbinval_range_check(vma, start, end, flushi, wbd); - - return 0; -} - -SYSCALL_DEFINE2(fp_udfiex_crtl, unsigned int, cmd, unsigned int, act) -{ -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) - int old_udf_iex; - - if (!used_math()) { - load_fpu(&init_fpuregs); - current->thread.fpu.UDF_IEX_trap = init_fpuregs.UDF_IEX_trap; - set_used_math(); - } - - old_udf_iex = current->thread.fpu.UDF_IEX_trap; - act &= (FPCSR_mskUDFE | FPCSR_mskIEXE); - - switch (cmd) { - case DISABLE_UDF_IEX_TRAP: - current->thread.fpu.UDF_IEX_trap &= ~act; - break; - case ENABLE_UDF_IEX_TRAP: - current->thread.fpu.UDF_IEX_trap |= act; - break; - case GET_UDF_IEX_TRAP: - break; - default: - return -EINVAL; - } - return old_udf_iex; -#else - return -ENOTSUPP; -#endif -} diff --git a/arch/nds32/kernel/syscall_table.c b/arch/nds32/kernel/syscall_table.c deleted file mode 100644 index 7879c061b87f..000000000000 --- a/arch/nds32/kernel/syscall_table.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/syscalls.h> -#include <linux/signal.h> -#include <linux/unistd.h> -#include <asm/syscalls.h> - -#undef __SYSCALL -#define __SYSCALL(nr, call) [nr] = (call), - -#define sys_rt_sigreturn sys_rt_sigreturn_wrapper -#define sys_fadvise64_64 sys_fadvise64_64_wrapper -void *sys_call_table[__NR_syscalls] __aligned(8192) = { - [0 ... __NR_syscalls - 1] = sys_ni_syscall, -#include <asm/unistd.h> -}; diff --git a/arch/nds32/kernel/time.c b/arch/nds32/kernel/time.c deleted file mode 100644 index 574a3d0a8539..000000000000 --- a/arch/nds32/kernel/time.c +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/clocksource.h> -#include <linux/of_clk.h> - -void __init time_init(void) -{ - of_clk_init(NULL); - timer_probe(); -} diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c deleted file mode 100644 index c0a8f3344fb9..000000000000 --- a/arch/nds32/kernel/traps.c +++ /dev/null @@ -1,354 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/module.h> -#include <linux/personality.h> -#include <linux/kallsyms.h> -#include <linux/hardirq.h> -#include <linux/kdebug.h> -#include <linux/sched/task_stack.h> -#include <linux/uaccess.h> -#include <linux/ftrace.h> - -#include <asm/proc-fns.h> -#include <asm/unistd.h> -#include <asm/fpu.h> - -#include <linux/ptrace.h> -#include <nds32_intrinsic.h> - -extern void show_pte(struct mm_struct *mm, unsigned long addr); - -/* - * Dump out the contents of some memory nicely... - */ -void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) -{ - unsigned long first; - int i; - - pr_emerg("%s(0x%08lx to 0x%08lx)\n", lvl, bottom, top); - - for (first = bottom & ~31; first < top; first += 32) { - unsigned long p; - char str[sizeof(" 12345678") * 8 + 1]; - - memset(str, ' ', sizeof(str)); - str[sizeof(str) - 1] = '\0'; - - for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { - if (p >= bottom && p < top) { - unsigned long val; - - if (get_kernel_nofault(val, - (unsigned long *)p) == 0) - sprintf(str + i * 9, " %08lx", val); - else - sprintf(str + i * 9, " ????????"); - } - } - pr_emerg("%s%04lx:%s\n", lvl, first & 0xffff, str); - } -} - -EXPORT_SYMBOL(dump_mem); - -#define LOOP_TIMES (100) -static void __dump(struct task_struct *tsk, unsigned long *base_reg, - const char *loglvl) -{ - unsigned long ret_addr; - int cnt = LOOP_TIMES, graph = 0; - printk("%sCall Trace:\n", loglvl); - if (!IS_ENABLED(CONFIG_FRAME_POINTER)) { - while (!kstack_end(base_reg)) { - ret_addr = *base_reg++; - if (__kernel_text_address(ret_addr)) { - ret_addr = ftrace_graph_ret_addr( - tsk, &graph, ret_addr, NULL); - print_ip_sym(loglvl, ret_addr); - } - if (--cnt < 0) - break; - } - } else { - while (!kstack_end((void *)base_reg) && - !((unsigned long)base_reg & 0x3) && - ((unsigned long)base_reg >= TASK_SIZE)) { - unsigned long next_fp; - ret_addr = base_reg[LP_OFFSET]; - next_fp = base_reg[FP_OFFSET]; - if (__kernel_text_address(ret_addr)) { - - ret_addr = ftrace_graph_ret_addr( - tsk, &graph, ret_addr, NULL); - print_ip_sym(loglvl, ret_addr); - } - if (--cnt < 0) - break; - base_reg = (unsigned long *)next_fp; - } - } - printk("%s\n", loglvl); -} - -void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) -{ - unsigned long *base_reg; - - if (!tsk) - tsk = current; - if (!IS_ENABLED(CONFIG_FRAME_POINTER)) { - if (tsk != current) - base_reg = (unsigned long *)(tsk->thread.cpu_context.sp); - else - __asm__ __volatile__("\tori\t%0, $sp, #0\n":"=r"(base_reg)); - } else { - if (tsk != current) - base_reg = (unsigned long *)(tsk->thread.cpu_context.fp); - else - __asm__ __volatile__("\tori\t%0, $fp, #0\n":"=r"(base_reg)); - } - __dump(tsk, base_reg, loglvl); - barrier(); -} - -DEFINE_SPINLOCK(die_lock); - -/* - * This function is protected against re-entrancy. - */ -void __noreturn die(const char *str, struct pt_regs *regs, int err) -{ - struct task_struct *tsk = current; - static int die_counter; - - console_verbose(); - spin_lock_irq(&die_lock); - bust_spinlocks(1); - - pr_emerg("Internal error: %s: %x [#%d]\n", str, err, ++die_counter); - print_modules(); - pr_emerg("CPU: %i\n", smp_processor_id()); - show_regs(regs); - pr_emerg("Process %s (pid: %d, stack limit = 0x%p)\n", - tsk->comm, tsk->pid, end_of_stack(tsk)); - - if (!user_mode(regs) || in_interrupt()) { - dump_mem("Stack: ", regs->sp, (regs->sp + PAGE_SIZE) & PAGE_MASK); - dump_stack(); - } - - bust_spinlocks(0); - spin_unlock_irq(&die_lock); - make_task_dead(SIGSEGV); -} - -EXPORT_SYMBOL(die); - -void die_if_kernel(const char *str, struct pt_regs *regs, int err) -{ - if (user_mode(regs)) - return; - - die(str, regs, err); -} - -int bad_syscall(int n, struct pt_regs *regs) -{ - if (current->personality != PER_LINUX) { - send_sig(SIGSEGV, current, 1); - return regs->uregs[0]; - } - - force_sig_fault(SIGILL, ILL_ILLTRP, - (void __user *)instruction_pointer(regs) - 4); - die_if_kernel("Oops - bad syscall", regs, n); - return regs->uregs[0]; -} - -void __pte_error(const char *file, int line, unsigned long val) -{ - pr_emerg("%s:%d: bad pte %08lx.\n", file, line, val); -} - -void __pmd_error(const char *file, int line, unsigned long val) -{ - pr_emerg("%s:%d: bad pmd %08lx.\n", file, line, val); -} - -void __pgd_error(const char *file, int line, unsigned long val) -{ - pr_emerg("%s:%d: bad pgd %08lx.\n", file, line, val); -} - -extern char *exception_vector, *exception_vector_end; -void __init early_trap_init(void) -{ - unsigned long ivb = 0; - unsigned long base = PAGE_OFFSET; - - memcpy((unsigned long *)base, (unsigned long *)&exception_vector, - ((unsigned long)&exception_vector_end - - (unsigned long)&exception_vector)); - ivb = __nds32__mfsr(NDS32_SR_IVB); - /* Check platform support. */ - if (((ivb & IVB_mskNIVIC) >> IVB_offNIVIC) < 2) - panic - ("IVIC mode is not allowed on the platform with interrupt controller\n"); - __nds32__mtsr((ivb & ~IVB_mskESZ) | (IVB_valESZ16 << IVB_offESZ) | - IVB_BASE, NDS32_SR_IVB); - __nds32__mtsr(INT_MASK_INITAIAL_VAL, NDS32_SR_INT_MASK); - - /* - * 0x800 = 128 vectors * 16byte. - * It should be enough to flush a page. - */ - cpu_cache_wbinval_page(base, true); -} - -static void send_sigtrap(struct pt_regs *regs, int error_code, int si_code) -{ - struct task_struct *tsk = current; - - tsk->thread.trap_no = ENTRY_DEBUG_RELATED; - tsk->thread.error_code = error_code; - - force_sig_fault(SIGTRAP, si_code, - (void __user *)instruction_pointer(regs)); -} - -void do_debug_trap(unsigned long entry, unsigned long addr, - unsigned long type, struct pt_regs *regs) -{ - if (notify_die(DIE_OOPS, "Oops", regs, addr, type, SIGTRAP) - == NOTIFY_STOP) - return; - - if (user_mode(regs)) { - /* trap_signal */ - send_sigtrap(regs, 0, TRAP_BRKPT); - } else { - /* kernel_trap */ - if (!fixup_exception(regs)) - die("unexpected kernel_trap", regs, 0); - } -} - -void unhandled_interruption(struct pt_regs *regs) -{ - pr_emerg("unhandled_interruption\n"); - show_regs(regs); - if (!user_mode(regs)) - make_task_dead(SIGKILL); - force_sig(SIGKILL); -} - -void unhandled_exceptions(unsigned long entry, unsigned long addr, - unsigned long type, struct pt_regs *regs) -{ - pr_emerg("Unhandled Exception: entry: %lx addr:%lx itype:%lx\n", entry, - addr, type); - show_regs(regs); - if (!user_mode(regs)) - make_task_dead(SIGKILL); - force_sig(SIGKILL); -} - -extern int do_page_fault(unsigned long entry, unsigned long addr, - unsigned int error_code, struct pt_regs *regs); - -/* - * 2:DEF dispatch for TLB MISC exception handler -*/ - -void do_dispatch_tlb_misc(unsigned long entry, unsigned long addr, - unsigned long type, struct pt_regs *regs) -{ - type = type & (ITYPE_mskINST | ITYPE_mskETYPE); - if ((type & ITYPE_mskETYPE) < 5) { - /* Permission exceptions */ - do_page_fault(entry, addr, type, regs); - } else - unhandled_exceptions(entry, addr, type, regs); -} - -void do_revinsn(struct pt_regs *regs) -{ - pr_emerg("Reserved Instruction\n"); - show_regs(regs); - if (!user_mode(regs)) - make_task_dead(SIGILL); - force_sig(SIGILL); -} - -#ifdef CONFIG_ALIGNMENT_TRAP -extern int unalign_access_mode; -extern int do_unaligned_access(unsigned long addr, struct pt_regs *regs); -#endif -void do_dispatch_general(unsigned long entry, unsigned long addr, - unsigned long itype, struct pt_regs *regs, - unsigned long oipc) -{ - unsigned int swid = itype >> ITYPE_offSWID; - unsigned long type = itype & (ITYPE_mskINST | ITYPE_mskETYPE); - if (type == ETYPE_ALIGNMENT_CHECK) { -#ifdef CONFIG_ALIGNMENT_TRAP - /* Alignment check */ - if (user_mode(regs) && unalign_access_mode) { - int ret; - ret = do_unaligned_access(addr, regs); - - if (ret == 0) - return; - - if (ret == -EFAULT) - pr_emerg - ("Unhandled unaligned access exception\n"); - } -#endif - do_page_fault(entry, addr, type, regs); - } else if (type == ETYPE_RESERVED_INSTRUCTION) { - /* Reserved instruction */ - do_revinsn(regs); - } else if (type == ETYPE_COPROCESSOR) { - /* Coprocessor */ -#if IS_ENABLED(CONFIG_FPU) - unsigned int fucop_exist = __nds32__mfsr(NDS32_SR_FUCOP_EXIST); - unsigned int cpid = ((itype & ITYPE_mskCPID) >> ITYPE_offCPID); - - if ((cpid == FPU_CPID) && - (fucop_exist & FUCOP_EXIST_mskCP0ISFPU)) { - unsigned int subtype = (itype & ITYPE_mskSTYPE); - - if (true == do_fpu_exception(subtype, regs)) - return; - } -#endif - unhandled_exceptions(entry, addr, type, regs); - } else if (type == ETYPE_TRAP && swid == SWID_RAISE_INTERRUPT_LEVEL) { - /* trap, used on v3 EDM target debugging workaround */ - /* - * DIPC(OIPC) is passed as parameter before - * interrupt is enabled, so the DIPC will not be corrupted - * even though interrupts are coming in - */ - /* - * 1. update ipc - * 2. update pt_regs ipc with oipc - * 3. update pt_regs ipsw (clear DEX) - */ - __asm__ volatile ("mtsr %0, $IPC\n\t"::"r" (oipc)); - regs->ipc = oipc; - if (regs->pipsw & PSW_mskDEX) { - pr_emerg - ("Nested Debug exception is possibly happened\n"); - pr_emerg("ipc:%08x pipc:%08x\n", - (unsigned int)regs->ipc, - (unsigned int)regs->pipc); - } - do_debug_trap(entry, addr, itype, regs); - regs->ipsw &= ~PSW_mskDEX; - } else - unhandled_exceptions(entry, addr, type, regs); -} diff --git a/arch/nds32/kernel/vdso.c b/arch/nds32/kernel/vdso.c deleted file mode 100644 index e16009a07971..000000000000 --- a/arch/nds32/kernel/vdso.c +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2012 ARM Limited -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/cache.h> -#include <linux/clocksource.h> -#include <linux/elf.h> -#include <linux/err.h> -#include <linux/errno.h> -#include <linux/gfp.h> -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/sched.h> -#include <linux/signal.h> -#include <linux/slab.h> -#include <linux/timekeeper_internal.h> -#include <linux/vmalloc.h> -#include <linux/random.h> - -#include <asm/cacheflush.h> -#include <asm/vdso.h> -#include <asm/vdso_datapage.h> -#include <asm/vdso_timer_info.h> -#include <asm/cache_info.h> -extern struct cache_info L1_cache_info[2]; -extern char vdso_start[], vdso_end[]; -static unsigned long vdso_pages __ro_after_init; -static unsigned long timer_mapping_base; - -struct timer_info_t timer_info = { - .cycle_count_down = true, - .mapping_base = EMPTY_TIMER_MAPPING, - .cycle_count_reg_offset = EMPTY_REG_OFFSET -}; -/* - * The vDSO data page. - */ -static struct page *no_pages[] = { NULL }; - -static union { - struct vdso_data data; - u8 page[PAGE_SIZE]; -} vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = &vdso_data_store.data; -static struct vm_special_mapping vdso_spec[2] __ro_after_init = { - { - .name = "[vvar]", - .pages = no_pages, - }, - { - .name = "[vdso]", - }, -}; - -static void get_timer_node_info(void) -{ - timer_mapping_base = timer_info.mapping_base; - vdso_data->cycle_count_offset = - timer_info.cycle_count_reg_offset; - vdso_data->cycle_count_down = - timer_info.cycle_count_down; -} - -static int __init vdso_init(void) -{ - int i; - struct page **vdso_pagelist; - - if (memcmp(vdso_start, "\177ELF", 4)) { - pr_err("vDSO is not a valid ELF object!\n"); - return -EINVAL; - } - /* Creat a timer io mapping to get clock cycles counter */ - get_timer_node_info(); - - vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; - pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", - vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data); - - /* Allocate the vDSO pagelist */ - vdso_pagelist = kcalloc(vdso_pages, sizeof(struct page *), GFP_KERNEL); - if (vdso_pagelist == NULL) - return -ENOMEM; - - for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i] = virt_to_page(vdso_start + i * PAGE_SIZE); - vdso_spec[1].pages = &vdso_pagelist[0]; - - return 0; -} - -arch_initcall(vdso_init); - -unsigned long inline vdso_random_addr(unsigned long vdso_mapping_len) -{ - unsigned long start = current->mm->mmap_base, end, offset, addr; - start = PAGE_ALIGN(start); - - /* Round the lowest possible end address up to a PMD boundary. */ - end = (start + vdso_mapping_len + PMD_SIZE - 1) & PMD_MASK; - if (end >= TASK_SIZE) - end = TASK_SIZE; - end -= vdso_mapping_len; - - if (end > start) { - offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1); - addr = start + (offset << PAGE_SHIFT); - } else { - addr = start; - } - return addr; -} - -int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) -{ - struct mm_struct *mm = current->mm; - unsigned long vdso_base, vdso_text_len, vdso_mapping_len; - struct vm_area_struct *vma; - unsigned long addr = 0; - pgprot_t prot; - int ret, vvar_page_num = 2; - - vdso_text_len = vdso_pages << PAGE_SHIFT; - - if(timer_mapping_base == EMPTY_VALUE) - vvar_page_num = 1; - /* Be sure to map the data page */ - vdso_mapping_len = vdso_text_len + vvar_page_num * PAGE_SIZE; -#ifdef CONFIG_CPU_CACHE_ALIASING - vdso_mapping_len += L1_cache_info[DCACHE].aliasing_num - 1; -#endif - - if (mmap_write_lock_killable(mm)) - return -EINTR; - - addr = vdso_random_addr(vdso_mapping_len); - vdso_base = get_unmapped_area(NULL, addr, vdso_mapping_len, 0, 0); - if (IS_ERR_VALUE(vdso_base)) { - ret = vdso_base; - goto up_fail; - } - -#ifdef CONFIG_CPU_CACHE_ALIASING - { - unsigned int aliasing_mask = - L1_cache_info[DCACHE].aliasing_mask; - unsigned int page_colour_ofs; - page_colour_ofs = ((unsigned int)vdso_data & aliasing_mask) - - (vdso_base & aliasing_mask); - vdso_base += page_colour_ofs & aliasing_mask; - } -#endif - - vma = _install_special_mapping(mm, vdso_base, vvar_page_num * PAGE_SIZE, - VM_READ | VM_MAYREAD, &vdso_spec[0]); - if (IS_ERR(vma)) { - ret = PTR_ERR(vma); - goto up_fail; - } - - /*Map vdata to user space */ - ret = io_remap_pfn_range(vma, vdso_base, - virt_to_phys(vdso_data) >> PAGE_SHIFT, - PAGE_SIZE, vma->vm_page_prot); - if (ret) - goto up_fail; - - /*Map timer to user space */ - vdso_base += PAGE_SIZE; - prot = __pgprot(_PAGE_V | _PAGE_M_UR_KR | _PAGE_D | _PAGE_C_DEV); - ret = io_remap_pfn_range(vma, vdso_base, timer_mapping_base >> PAGE_SHIFT, - PAGE_SIZE, prot); - if (ret) - goto up_fail; - - /*Map vdso to user space */ - vdso_base += PAGE_SIZE; - mm->context.vdso = (void *)vdso_base; - vma = _install_special_mapping(mm, vdso_base, vdso_text_len, - VM_READ | VM_EXEC | - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, - &vdso_spec[1]); - if (IS_ERR(vma)) { - ret = PTR_ERR(vma); - goto up_fail; - } - - mmap_write_unlock(mm); - return 0; - -up_fail: - mm->context.vdso = NULL; - mmap_write_unlock(mm); - return ret; -} - -static void vdso_write_begin(struct vdso_data *vdata) -{ - ++vdso_data->seq_count; - smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */ -} - -static void vdso_write_end(struct vdso_data *vdata) -{ - smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */ - ++vdso_data->seq_count; -} - -void update_vsyscall(struct timekeeper *tk) -{ - vdso_write_begin(vdso_data); - vdso_data->cs_mask = tk->tkr_mono.mask; - vdso_data->cs_mult = tk->tkr_mono.mult; - vdso_data->cs_shift = tk->tkr_mono.shift; - vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; - vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec; - vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; - vdso_data->xtime_clock_sec = tk->xtime_sec; - vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; - vdso_data->xtime_coarse_sec = tk->xtime_sec; - vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >> - tk->tkr_mono.shift; - vdso_data->hrtimer_res = hrtimer_resolution; - vdso_write_end(vdso_data); -} - -void update_vsyscall_tz(void) -{ - vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; - vdso_data->tz_dsttime = sys_tz.tz_dsttime; -} diff --git a/arch/nds32/kernel/vdso/.gitignore b/arch/nds32/kernel/vdso/.gitignore deleted file mode 100644 index 652e31d82582..000000000000 --- a/arch/nds32/kernel/vdso/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -vdso.lds diff --git a/arch/nds32/kernel/vdso/Makefile b/arch/nds32/kernel/vdso/Makefile deleted file mode 100644 index 55df25ef0057..000000000000 --- a/arch/nds32/kernel/vdso/Makefile +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Building a vDSO image for AArch64. -# -# Author: Will Deacon <will.deacon@arm.com> -# Heavily based on the vDSO Makefiles for other archs. -# - -obj-vdso := note.o datapage.o sigreturn.o gettimeofday.o - -# Build rules -targets := $(obj-vdso) vdso.so vdso.so.dbg -obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) - -ccflags-y := -shared -fno-common -fno-builtin -nostdlib -fPIC -Wl,-shared -g \ - -Wl,-soname=linux-vdso.so.1 -Wl,--hash-style=sysv - -# Disable gcov profiling for VDSO code -GCOV_PROFILE := n - - -obj-y += vdso.o -targets += vdso.lds -CPPFLAGS_vdso.lds += -P -C -U$(ARCH) - -# Force dependency -$(obj)/vdso.o : $(obj)/vdso.so - -# Link rule for the .so file, .lds has to be first -$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE - $(call if_changed,vdsold) - - -# Strip rule for the .so file -$(obj)/%.so: OBJCOPYFLAGS := -S -$(obj)/%.so: $(obj)/%.so.dbg FORCE - $(call if_changed,objcopy) - -# Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh -quiet_cmd_vdsosym = VDSOSYM $@ - cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ - -include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE - $(call if_changed,vdsosym) - - - -# Assembly rules for the .S files - -sigreturn.o : sigreturn.S - $(call if_changed_dep,vdsoas) - -note.o : note.S - $(call if_changed_dep,vdsoas) - -datapage.o : datapage.S - $(call if_changed_dep,vdsoas) - -gettimeofday.o : gettimeofday.c FORCE - $(call if_changed_dep,vdsocc) - -# Actual build commands -quiet_cmd_vdsold = VDSOL $@ - cmd_vdsold = $(CC) $(c_flags) -Wl,-n -Wl,-T $(real-prereqs) -o $@ -quiet_cmd_vdsoas = VDSOA $@ - cmd_vdsoas = $(CC) $(a_flags) -c -o $@ $< -quiet_cmd_vdsocc = VDSOA $@ - cmd_vdsocc = $(CC) $(c_flags) -c -o $@ $< - -# Install commands for the unstripped file -quiet_cmd_vdso_install = INSTALL $@ - cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ - -vdso.so: $(obj)/vdso.so.dbg - @mkdir -p $(MODLIB)/vdso - $(call cmd,vdso_install) - -vdso_install: vdso.so diff --git a/arch/nds32/kernel/vdso/datapage.S b/arch/nds32/kernel/vdso/datapage.S deleted file mode 100644 index 4a62c3cab1c8..000000000000 --- a/arch/nds32/kernel/vdso/datapage.S +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/page.h> - -ENTRY(__get_timerpage) - sethi $r0, hi20(. + PAGE_SIZE + 8) - ori $r0, $r0, lo12(. + PAGE_SIZE + 4) - mfusr $r1, $pc - sub $r0, $r1, $r0 - ret -ENDPROC(__get_timerpage) - -ENTRY(__get_datapage) - sethi $r0, hi20(. + 2*PAGE_SIZE + 8) - ori $r0, $r0, lo12(. + 2*PAGE_SIZE + 4) - mfusr $r1, $pc - sub $r0, $r1, $r0 - ret -ENDPROC(__get_datapage) diff --git a/arch/nds32/kernel/vdso/gettimeofday.c b/arch/nds32/kernel/vdso/gettimeofday.c deleted file mode 100644 index 9ec03cf0ec54..000000000000 --- a/arch/nds32/kernel/vdso/gettimeofday.c +++ /dev/null @@ -1,269 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/compiler.h> -#include <linux/hrtimer.h> -#include <linux/time.h> -#include <asm/io.h> -#include <asm/barrier.h> -#include <asm/bug.h> -#include <asm/page.h> -#include <asm/unistd.h> -#include <asm/vdso_datapage.h> -#include <asm/vdso_timer_info.h> -#include <asm/asm-offsets.h> - -#define X(x) #x -#define Y(x) X(x) - -extern struct vdso_data *__get_datapage(void); -extern struct vdso_data *__get_timerpage(void); - -static notrace unsigned int __vdso_read_begin(const struct vdso_data *vdata) -{ - u32 seq; -repeat: - seq = READ_ONCE(vdata->seq_count); - if (seq & 1) { - cpu_relax(); - goto repeat; - } - return seq; -} - -static notrace unsigned int vdso_read_begin(const struct vdso_data *vdata) -{ - unsigned int seq; - - seq = __vdso_read_begin(vdata); - - smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */ - return seq; -} - -static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start) -{ - smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */ - return vdata->seq_count != start; -} - -static notrace long clock_gettime_fallback(clockid_t _clkid, - struct __kernel_old_timespec *_ts) -{ - register struct __kernel_old_timespec *ts asm("$r1") = _ts; - register clockid_t clkid asm("$r0") = _clkid; - register long ret asm("$r0"); - - asm volatile ("movi $r15, %3\n" - "syscall 0x0\n" - :"=r" (ret) - :"r"(clkid), "r"(ts), "i"(__NR_clock_gettime) - :"$r15", "memory"); - - return ret; -} - -static notrace int do_realtime_coarse(struct __kernel_old_timespec *ts, - struct vdso_data *vdata) -{ - u32 seq; - - do { - seq = vdso_read_begin(vdata); - - ts->tv_sec = vdata->xtime_coarse_sec; - ts->tv_nsec = vdata->xtime_coarse_nsec; - - } while (vdso_read_retry(vdata, seq)); - return 0; -} - -static notrace int do_monotonic_coarse(struct __kernel_old_timespec *ts, - struct vdso_data *vdata) -{ - u32 seq; - u64 ns; - - do { - seq = vdso_read_begin(vdata); - - ts->tv_sec = vdata->xtime_coarse_sec + vdata->wtm_clock_sec; - ns = vdata->xtime_coarse_nsec + vdata->wtm_clock_nsec; - - } while (vdso_read_retry(vdata, seq)); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; -} - -static notrace inline u64 vgetsns(struct vdso_data *vdso) -{ - u32 cycle_now; - u32 cycle_delta; - u32 *timer_cycle_base; - - timer_cycle_base = - (u32 *) ((char *)__get_timerpage() + vdso->cycle_count_offset); - cycle_now = readl_relaxed(timer_cycle_base); - if (true == vdso->cycle_count_down) - cycle_now = ~(*timer_cycle_base); - cycle_delta = cycle_now - (u32) vdso->cs_cycle_last; - return ((u64) cycle_delta & vdso->cs_mask) * vdso->cs_mult; -} - -static notrace int do_realtime(struct __kernel_old_timespec *ts, struct vdso_data *vdata) -{ - unsigned count; - u64 ns; - do { - count = vdso_read_begin(vdata); - ts->tv_sec = vdata->xtime_clock_sec; - ns = vdata->xtime_clock_nsec; - ns += vgetsns(vdata); - ns >>= vdata->cs_shift; - } while (vdso_read_retry(vdata, count)); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; -} - -static notrace int do_monotonic(struct __kernel_old_timespec *ts, struct vdso_data *vdata) -{ - u64 ns; - u32 seq; - - do { - seq = vdso_read_begin(vdata); - - ts->tv_sec = vdata->xtime_clock_sec; - ns = vdata->xtime_clock_nsec; - ns += vgetsns(vdata); - ns >>= vdata->cs_shift; - - ts->tv_sec += vdata->wtm_clock_sec; - ns += vdata->wtm_clock_nsec; - - } while (vdso_read_retry(vdata, seq)); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; -} - -notrace int __vdso_clock_gettime(clockid_t clkid, struct __kernel_old_timespec *ts) -{ - struct vdso_data *vdata; - int ret = -1; - - vdata = __get_datapage(); - if (vdata->cycle_count_offset == EMPTY_REG_OFFSET) - return clock_gettime_fallback(clkid, ts); - - switch (clkid) { - case CLOCK_REALTIME_COARSE: - ret = do_realtime_coarse(ts, vdata); - break; - case CLOCK_MONOTONIC_COARSE: - ret = do_monotonic_coarse(ts, vdata); - break; - case CLOCK_REALTIME: - ret = do_realtime(ts, vdata); - break; - case CLOCK_MONOTONIC: - ret = do_monotonic(ts, vdata); - break; - default: - break; - } - - if (ret) - ret = clock_gettime_fallback(clkid, ts); - - return ret; -} - -static notrace int clock_getres_fallback(clockid_t _clk_id, - struct __kernel_old_timespec *_res) -{ - register clockid_t clk_id asm("$r0") = _clk_id; - register struct __kernel_old_timespec *res asm("$r1") = _res; - register int ret asm("$r0"); - - asm volatile ("movi $r15, %3\n" - "syscall 0x0\n" - :"=r" (ret) - :"r"(clk_id), "r"(res), "i"(__NR_clock_getres) - :"$r15", "memory"); - - return ret; -} - -notrace int __vdso_clock_getres(clockid_t clk_id, struct __kernel_old_timespec *res) -{ - struct vdso_data *vdata = __get_datapage(); - - if (res == NULL) - return 0; - switch (clk_id) { - case CLOCK_REALTIME: - case CLOCK_MONOTONIC: - case CLOCK_MONOTONIC_RAW: - res->tv_sec = 0; - res->tv_nsec = vdata->hrtimer_res; - break; - case CLOCK_REALTIME_COARSE: - case CLOCK_MONOTONIC_COARSE: - res->tv_sec = 0; - res->tv_nsec = CLOCK_COARSE_RES; - break; - default: - return clock_getres_fallback(clk_id, res); - } - return 0; -} - -static notrace inline int gettimeofday_fallback(struct __kernel_old_timeval *_tv, - struct timezone *_tz) -{ - register struct __kernel_old_timeval *tv asm("$r0") = _tv; - register struct timezone *tz asm("$r1") = _tz; - register int ret asm("$r0"); - - asm volatile ("movi $r15, %3\n" - "syscall 0x0\n" - :"=r" (ret) - :"r"(tv), "r"(tz), "i"(__NR_gettimeofday) - :"$r15", "memory"); - - return ret; -} - -notrace int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) -{ - struct __kernel_old_timespec ts; - struct vdso_data *vdata; - int ret; - - vdata = __get_datapage(); - - if (vdata->cycle_count_offset == EMPTY_REG_OFFSET) - return gettimeofday_fallback(tv, tz); - - ret = do_realtime(&ts, vdata); - - if (tv) { - tv->tv_sec = ts.tv_sec; - tv->tv_usec = ts.tv_nsec / 1000; - } - if (tz) { - tz->tz_minuteswest = vdata->tz_minuteswest; - tz->tz_dsttime = vdata->tz_dsttime; - } - - return ret; -} diff --git a/arch/nds32/kernel/vdso/note.S b/arch/nds32/kernel/vdso/note.S deleted file mode 100644 index 0aeaa19b05f0..000000000000 --- a/arch/nds32/kernel/vdso/note.S +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2012 ARM Limited -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/uts.h> -#include <linux/version.h> -#include <linux/elfnote.h> - -ELFNOTE_START(Linux, 0, "a") - .long LINUX_VERSION_CODE -ELFNOTE_END diff --git a/arch/nds32/kernel/vdso/sigreturn.S b/arch/nds32/kernel/vdso/sigreturn.S deleted file mode 100644 index 67e4d1d1612a..000000000000 --- a/arch/nds32/kernel/vdso/sigreturn.S +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2012 ARM Limited -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text - -ENTRY(__kernel_rt_sigreturn) - .cfi_startproc - movi $r15, __NR_rt_sigreturn - /* - * The SWID of syscall should be __NR_rt_sigreturn to synchronize - * the unwinding scheme in gcc - */ - syscall __NR_rt_sigreturn - .cfi_endproc -ENDPROC(__kernel_rt_sigreturn) diff --git a/arch/nds32/kernel/vdso/vdso.S b/arch/nds32/kernel/vdso/vdso.S deleted file mode 100644 index 16737c11e55b..000000000000 --- a/arch/nds32/kernel/vdso/vdso.S +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2012 ARM Limited -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/init.h> -#include <linux/linkage.h> -#include <linux/const.h> -#include <asm/page.h> - - .globl vdso_start, vdso_end - .section .rodata - .balign PAGE_SIZE -vdso_start: - .incbin "arch/nds32/kernel/vdso/vdso.so" - .balign PAGE_SIZE -vdso_end: - - .previous diff --git a/arch/nds32/kernel/vdso/vdso.lds.S b/arch/nds32/kernel/vdso/vdso.lds.S deleted file mode 100644 index 1f2b16004594..000000000000 --- a/arch/nds32/kernel/vdso/vdso.lds.S +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: GPL-2.0 - * Copyright (C) 2005-2017 Andes Technology Corporation - */ - - -#include <linux/const.h> -#include <asm/page.h> -#include <asm/vdso.h> - -OUTPUT_ARCH(nds32) - -SECTIONS -{ - . = SIZEOF_HEADERS; - - .hash : { *(.hash) } :text - .gnu.hash : { *(.gnu.hash) } - .dynsym : { *(.dynsym) } - .dynstr : { *(.dynstr) } - .gnu.version : { *(.gnu.version) } - .gnu.version_d : { *(.gnu.version_d) } - .gnu.version_r : { *(.gnu.version_r) } - - .note : { *(.note.*) } :text :note - - - .text : { *(.text*) } :text - - .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr - .eh_frame : { KEEP (*(.eh_frame)) } :text - - .dynamic : { *(.dynamic) } :text :dynamic - - .rodata : { *(.rodata*) } :text - - - /DISCARD/ : { - *(.note.GNU-stack) - *(.data .data.* .gnu.linkonce.d.* .sdata*) - *(.bss .sbss .dynbss .dynsbss) - } -} - -/* - * We must supply the ELF program headers explicitly to get just one - * PT_LOAD segment, and set the flags explicitly to make segments read-only. - */ -PHDRS -{ - text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ - dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ - note PT_NOTE FLAGS(4); /* PF_R */ - eh_frame_hdr PT_GNU_EH_FRAME; -} - -/* - * This controls what symbols we export from the DSO. - */ -VERSION -{ - LINUX_4 { - global: - __kernel_rt_sigreturn; - __vdso_gettimeofday; - __vdso_clock_getres; - __vdso_clock_gettime; - local: *; - }; -} - -/* - * Make the rt_sigreturn code visible to the kernel. - */ -VDSO_rt_sigtramp = __kernel_rt_sigreturn; diff --git a/arch/nds32/kernel/vmlinux.lds.S b/arch/nds32/kernel/vmlinux.lds.S deleted file mode 100644 index 6a91b965fb1e..000000000000 --- a/arch/nds32/kernel/vmlinux.lds.S +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <asm/page.h> -#include <asm/thread_info.h> -#include <asm/cache.h> -#include <asm/memory.h> - -#define LOAD_OFFSET (PAGE_OFFSET - PHYS_OFFSET) -#include <asm-generic/vmlinux.lds.h> - -OUTPUT_ARCH(nds32) -ENTRY(_stext_lma) -jiffies = jiffies_64; - -#if defined(CONFIG_GCOV_KERNEL) -#define NDS32_EXIT_KEEP(x) x -#else -#define NDS32_EXIT_KEEP(x) -#endif - -SECTIONS -{ - _stext_lma = TEXTADDR - LOAD_OFFSET; - . = TEXTADDR; - __init_begin = .; - HEAD_TEXT_SECTION - .exit.text : { - NDS32_EXIT_KEEP(EXIT_TEXT) - } - INIT_TEXT_SECTION(PAGE_SIZE) - INIT_DATA_SECTION(16) - .exit.data : { - NDS32_EXIT_KEEP(EXIT_DATA) - } - PERCPU_SECTION(L1_CACHE_BYTES) - __init_end = .; - - . = ALIGN(PAGE_SIZE); - _stext = .; - /* Real text segment */ - .text : AT(ADDR(.text) - LOAD_OFFSET) { - _text = .; /* Text and read-only data */ - TEXT_TEXT - SCHED_TEXT - CPUIDLE_TEXT - LOCK_TEXT - KPROBES_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - *(.fixup) - } - - _etext = .; /* End of text and rodata section */ - - _sdata = .; - RO_DATA(PAGE_SIZE) - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) - _edata = .; - - EXCEPTION_TABLE(16) - BSS_SECTION(4, 4, 4) - _end = .; - - STABS_DEBUG - DWARF_DEBUG - ELF_DETAILS - - DISCARDS -} diff --git a/arch/nds32/lib/Makefile b/arch/nds32/lib/Makefile deleted file mode 100644 index dddbc15d6b37..000000000000 --- a/arch/nds32/lib/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -lib-y := copy_page.o memcpy.o memmove.o \ - memset.o memzero.o \ - copy_from_user.o copy_to_user.o clear_user.o diff --git a/arch/nds32/lib/clear_user.S b/arch/nds32/lib/clear_user.S deleted file mode 100644 index 805dfcd25bf8..000000000000 --- a/arch/nds32/lib/clear_user.S +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/errno.h> - -/* Prototype: int __arch_clear_user(void *addr, size_t sz) - * Purpose : clear some user memory - * Params : addr - user memory address to clear - * : sz - number of bytes to clear - * Returns : number of bytes NOT cleared - */ - .text - .align 5 -ENTRY(__arch_clear_user) - add $r5, $r0, $r1 - beqz $r1, clear_exit - xor $p1, $p1, $p1 ! Use $p1=0 to clear mem - srli $p0, $r1, #2 ! $p0 = number of word to clear - andi $r1, $r1, #3 ! Bytes less than a word to copy - beqz $p0, byte_clear ! Only less than a word to clear -word_clear: -USER( smw.bim,$p1, [$r0], $p1) ! Clear the word - addi $p0, $p0, #-1 ! Decrease word count - bnez $p0, word_clear ! Continue looping to clear all words - beqz $r1, clear_exit ! No left bytes to copy -byte_clear: -USER( sbi.bi, $p1, [$r0], #1) ! Clear the byte - addi $r1, $r1, #-1 ! Decrease byte count - bnez $r1, byte_clear ! Continue looping to clear all left bytes -clear_exit: - move $r0, $r1 ! Set return value - ret - - .section .fixup,"ax" - .align 0 -9001: - sub $r0, $r5, $r0 ! Bytes left to copy - ret - .previous -ENDPROC(__arch_clear_user) diff --git a/arch/nds32/lib/copy_from_user.S b/arch/nds32/lib/copy_from_user.S deleted file mode 100644 index ad1857b20067..000000000000 --- a/arch/nds32/lib/copy_from_user.S +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/errno.h> - -.macro lbi1 dst, addr, adj -USER( lbi.bi, \dst, [\addr], \adj) -.endm - -.macro sbi1 src, addr, adj -sbi.bi \src, [\addr], \adj -.endm - -.macro lmw1 start_reg, addr, end_reg -USER( lmw.bim, \start_reg, [\addr], \end_reg) -.endm - -.macro smw1 start_reg, addr, end_reg -smw.bim \start_reg, [\addr], \end_reg -.endm - - -/* Prototype: int __arch_copy_from_user(void *to, const char *from, size_t n) - * Purpose : copy a block from user memory to kernel memory - * Params : to - kernel memory - * : from - user memory - * : n - number of bytes to copy - * Returns : Number of bytes NOT copied. - */ - -.text -ENTRY(__arch_copy_from_user) - add $r5, $r0, $r2 -#include "copy_template.S" - move $r0, $r2 - ret -.section .fixup,"ax" -.align 2 -9001: - sub $r0, $r5, $r0 - ret -.previous -ENDPROC(__arch_copy_from_user) diff --git a/arch/nds32/lib/copy_page.S b/arch/nds32/lib/copy_page.S deleted file mode 100644 index f8701ed161a8..000000000000 --- a/arch/nds32/lib/copy_page.S +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/export.h> -#include <asm/page.h> - - .text -ENTRY(copy_page) - pushm $r2, $r10 - movi $r2, PAGE_SIZE >> 5 -.Lcopy_loop: - lmw.bim $r3, [$r1], $r10 - smw.bim $r3, [$r0], $r10 - subi45 $r2, #1 - bnez38 $r2, .Lcopy_loop - popm $r2, $r10 - ret -ENDPROC(copy_page) -EXPORT_SYMBOL(copy_page) - -ENTRY(clear_page) - pushm $r1, $r9 - movi $r1, PAGE_SIZE >> 5 - movi55 $r2, #0 - movi55 $r3, #0 - movi55 $r4, #0 - movi55 $r5, #0 - movi55 $r6, #0 - movi55 $r7, #0 - movi55 $r8, #0 - movi55 $r9, #0 -.Lclear_loop: - smw.bim $r2, [$r0], $r9 - subi45 $r1, #1 - bnez38 $r1, .Lclear_loop - popm $r1, $r9 - ret -ENDPROC(clear_page) -EXPORT_SYMBOL(clear_page) diff --git a/arch/nds32/lib/copy_template.S b/arch/nds32/lib/copy_template.S deleted file mode 100644 index 3a9a2de468c2..000000000000 --- a/arch/nds32/lib/copy_template.S +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - - - beq $r1, $r0, quit_memcpy - beqz $r2, quit_memcpy - srli $r3, $r2, #5 ! check if len < cache-line size 32 - beqz $r3, word_copy_entry - andi $r4, $r0, #0x3 ! check byte-align - beqz $r4, unalign_word_copy_entry - - addi $r4, $r4,#-4 - abs $r4, $r4 ! check how many un-align byte to copy - sub $r2, $r2, $r4 ! update $R2 - -unalign_byte_copy: - lbi1 $r3, $r1, #1 - addi $r4, $r4, #-1 - sbi1 $r3, $r0, #1 - bnez $r4, unalign_byte_copy - beqz $r2, quit_memcpy - -unalign_word_copy_entry: - andi $r3, $r0, 0x1f ! check cache-line unaligncount - beqz $r3, cache_copy - - addi $r3, $r3, #-32 - abs $r3, $r3 - sub $r2, $r2, $r3 ! update $R2 - -unalign_word_copy: - lmw1 $r4, $r1, $r4 - addi $r3, $r3, #-4 - smw1 $r4, $r0, $r4 - bnez $r3, unalign_word_copy - beqz $r2, quit_memcpy - - addi $r3, $r2, #-32 ! to check $r2< cache_line , than go to word_copy - bltz $r3, word_copy_entry -cache_copy: - srli $r3, $r2, #5 - beqz $r3, word_copy_entry -3: - lmw1 $r17, $r1, $r24 - addi $r3, $r3, #-1 - smw1 $r17, $r0, $r24 - bnez $r3, 3b - -word_copy_entry: - andi $r2, $r2, #31 - - beqz $r2, quit_memcpy -5: - srli $r3, $r2, #2 - beqz $r3, byte_copy -word_copy: - lmw1 $r4, $r1, $r4 - addi $r3, $r3, #-1 - smw1 $r4, $r0, $r4 - bnez $r3, word_copy - andi $r2, $r2, #3 - beqz $r2, quit_memcpy -byte_copy: - lbi1 $r3, $r1, #1 - addi $r2, $r2, #-1 - - sbi1 $r3, $r0, #1 - bnez $r2, byte_copy -quit_memcpy: diff --git a/arch/nds32/lib/copy_to_user.S b/arch/nds32/lib/copy_to_user.S deleted file mode 100644 index 3230044dcfb8..000000000000 --- a/arch/nds32/lib/copy_to_user.S +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/errno.h> - -.macro lbi1 dst, addr, adj -lbi.bi \dst, [\addr], \adj -.endm - -.macro sbi1 src, addr, adj -USER( sbi.bi, \src, [\addr], \adj) -.endm - -.macro lmw1 start_reg, addr, end_reg -lmw.bim \start_reg, [\addr], \end_reg -.endm - -.macro smw1 start_reg, addr, end_reg -USER( smw.bim, \start_reg, [\addr], \end_reg) -.endm - - -/* Prototype: int __arch_copy_to_user(void *to, const char *from, size_t n) - * Purpose : copy a block to user memory from kernel memory - * Params : to - user memory - * : from - kernel memory - * : n - number of bytes to copy - * Returns : Number of bytes NOT copied. - */ - -.text -ENTRY(__arch_copy_to_user) - add $r5, $r0, $r2 -#include "copy_template.S" - move $r0, $r2 - ret -.section .fixup,"ax" -.align 2 -9001: - sub $r0, $r5, $r0 - ret -.previous -ENDPROC(__arch_copy_to_user) diff --git a/arch/nds32/lib/memcpy.S b/arch/nds32/lib/memcpy.S deleted file mode 100644 index a2345ea721e4..000000000000 --- a/arch/nds32/lib/memcpy.S +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> - - -.macro lbi1 dst, addr, adj -lbi.bi \dst, [\addr], \adj -.endm - -.macro sbi1 src, addr, adj -sbi.bi \src, [\addr], \adj -.endm - -.macro lmw1 start_reg, addr, end_reg -lmw.bim \start_reg, [\addr], \end_reg -.endm - -.macro smw1 start_reg, addr, end_reg -smw.bim \start_reg, [\addr], \end_reg -.endm - -.text -ENTRY(memcpy) - move $r5, $r0 -#include "copy_template.S" - move $r0, $r5 - ret - -ENDPROC(memcpy) diff --git a/arch/nds32/lib/memmove.S b/arch/nds32/lib/memmove.S deleted file mode 100644 index c823aada2271..000000000000 --- a/arch/nds32/lib/memmove.S +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> - -/* - void *memmove(void *dst, const void *src, int n); - - dst: $r0 - src: $r1 - n : $r2 - ret: $r0 - pointer to the memory area dst. -*/ - .text - -ENTRY(memmove) - move $r5, $r0 ! Set return value = det - beq $r0, $r1, exit_memcpy ! Exit when det = src - beqz $r2, exit_memcpy ! Exit when n = 0 - pushm $t0, $t1 ! Save reg - srli $p1, $r2, #2 ! $p1 is how many words to copy - - ! Avoid data lost when memory overlap - ! Copy data reversely when src < dst - slt $p0, $r0, $r1 ! check if $r0 < $r1 - beqz $p0, do_reverse ! branch if dst > src - - ! No reverse, dst < src - andi $r2, $r2, #3 ! How many bytes are less than a word - li $t0, #1 ! Determining copy direction in byte_cpy - beqz $p1, byte_cpy ! When n is less than a word - -word_cpy: - lmw.bim $p0, [$r1], $p0 ! Read a word from src - addi $p1, $p1, #-1 ! How many words left to copy - smw.bim $p0, [$r0], $p0 ! Copy the word to det - bnez $p1, word_cpy ! If remained words > 0 - beqz $r2, end_memcpy ! No left bytes to copy - b byte_cpy - -do_reverse: - add $r0, $r0, $r2 ! Start with the end of $r0 - add $r1, $r1, $r2 ! Start with the end of $r1 - andi $r2, $r2, #3 ! How many bytes are less than a word - li $t0, #-1 ! Determining copy direction in byte_cpy - beqz $p1, reverse_byte_cpy ! When n is less than a word - -reverse_word_cpy: - lmw.adm $p0, [$r1], $p0 ! Read a word from src - addi $p1, $p1, #-1 ! How many words left to copy - smw.adm $p0, [$r0], $p0 ! Copy the word to det - bnez $p1, reverse_word_cpy ! If remained words > 0 - beqz $r2, end_memcpy ! No left bytes to copy - -reverse_byte_cpy: - addi $r0, $r0, #-1 - addi $r1, $r1, #-1 -byte_cpy: ! Less than 4 bytes to copy now - lb.bi $p0, [$r1], $t0 ! Read a byte from src - addi $r2, $r2, #-1 ! How many bytes left to copy - sb.bi $p0, [$r0], $t0 ! copy the byte to det - bnez $r2, byte_cpy ! If remained bytes > 0 - -end_memcpy: - popm $t0, $t1 -exit_memcpy: - move $r0, $r5 - ret - -ENDPROC(memmove) diff --git a/arch/nds32/lib/memset.S b/arch/nds32/lib/memset.S deleted file mode 100644 index 193cb6ce21a9..000000000000 --- a/arch/nds32/lib/memset.S +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> - - .text -ENTRY(memset) - move $r5, $r0 ! Return value - beqz $r2, end_memset ! Exit when len = 0 - srli $p1, $r2, 2 ! $p1 is how many words to copy - andi $r2, $r2, 3 ! How many bytes are less than a word - beqz $p1, byte_set ! When n is less than a word - - ! set $r1 from ??????ab to abababab - andi $r1, $r1, #0x00ff ! $r1 = 000000ab - slli $p0, $r1, #8 ! $p0 = 0000ab00 - or $r1, $r1, $p0 ! $r1 = 0000abab - slli $p0, $r1, #16 ! $p0 = abab0000 - or $r1, $r1, $p0 ! $r1 = abababab -word_set: - addi $p1, $p1, #-1 ! How many words left to copy - smw.bim $r1, [$r0], $r1 ! Copy the word to det - bnez $p1, word_set ! Still words to set, continue looping - beqz $r2, end_memset ! No left byte to set -byte_set: ! Less than 4 bytes left to set - addi $r2, $r2, #-1 ! Decrease len by 1 - sbi.bi $r1, [$r0], #1 ! Set data of the next byte to $r1 - bnez $r2, byte_set ! Still bytes left to set -end_memset: - move $r0, $r5 - ret - -ENDPROC(memset) diff --git a/arch/nds32/lib/memzero.S b/arch/nds32/lib/memzero.S deleted file mode 100644 index f055972c9343..000000000000 --- a/arch/nds32/lib/memzero.S +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/linkage.h> - - .text -ENTRY(memzero) - beqz $r1, 1f - push $lp - move $r2, $r1 - move $r1, #0 - push $r0 - bal memset - pop $r0 - pop $lp -1: - ret -ENDPROC(memzero) diff --git a/arch/nds32/math-emu/Makefile b/arch/nds32/math-emu/Makefile deleted file mode 100644 index 3bed7e5d5d05..000000000000 --- a/arch/nds32/math-emu/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the Linux/nds32 kernel FPU emulation. -# - -obj-y := fpuemu.o \ - fdivd.o fmuld.o fsubd.o faddd.o fs2d.o fsqrtd.o fcmpd.o fnegs.o \ - fd2si.o fd2ui.o fd2siz.o fd2uiz.o fsi2d.o fui2d.o \ - fdivs.o fmuls.o fsubs.o fadds.o fd2s.o fsqrts.o fcmps.o fnegd.o \ - fs2si.o fs2ui.o fs2siz.o fs2uiz.o fsi2s.o fui2s.o diff --git a/arch/nds32/math-emu/faddd.c b/arch/nds32/math-emu/faddd.c deleted file mode 100644 index f7fd4e3c3904..000000000000 --- a/arch/nds32/math-emu/faddd.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -void faddd(void *ft, void *fa, void *fb) -{ - FP_DECL_D(A); - FP_DECL_D(B); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - FP_UNPACK_DP(B, fb); - - FP_ADD_D(R, A, B); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/math-emu/fadds.c b/arch/nds32/math-emu/fadds.c deleted file mode 100644 index f5af6ca8cca5..000000000000 --- a/arch/nds32/math-emu/fadds.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fadds(void *ft, void *fa, void *fb) -{ - FP_DECL_S(A); - FP_DECL_S(B); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - FP_UNPACK_SP(B, fb); - - FP_ADD_S(R, A, B); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/math-emu/fcmpd.c b/arch/nds32/math-emu/fcmpd.c deleted file mode 100644 index 0ea225abe880..000000000000 --- a/arch/nds32/math-emu/fcmpd.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -int fcmpd(void *ft, void *fa, void *fb, int cmpop) -{ - FP_DECL_D(A); - FP_DECL_D(B); - FP_DECL_EX; - long cmp; - - FP_UNPACK_DP(A, fa); - FP_UNPACK_DP(B, fb); - - FP_CMP_D(cmp, A, B, SF_CUN); - cmp += 2; - if (cmp == SF_CGT) - *(long *)ft = 0; - else - *(long *)ft = (cmp & cmpop) ? 1 : 0; - - return 0; -} diff --git a/arch/nds32/math-emu/fcmps.c b/arch/nds32/math-emu/fcmps.c deleted file mode 100644 index 681480758213..000000000000 --- a/arch/nds32/math-emu/fcmps.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -int fcmps(void *ft, void *fa, void *fb, int cmpop) -{ - FP_DECL_S(A); - FP_DECL_S(B); - FP_DECL_EX; - long cmp; - - FP_UNPACK_SP(A, fa); - FP_UNPACK_SP(B, fb); - - FP_CMP_S(cmp, A, B, SF_CUN); - cmp += 2; - if (cmp == SF_CGT) - *(int *)ft = 0x0; - else - *(int *)ft = (cmp & cmpop) ? 0x1 : 0x0; - - return 0; -} diff --git a/arch/nds32/math-emu/fd2s.c b/arch/nds32/math-emu/fd2s.c deleted file mode 100644 index 1328371e8170..000000000000 --- a/arch/nds32/math-emu/fd2s.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/double.h> -#include <math-emu/single.h> -#include <math-emu/soft-fp.h> -void fd2s(void *ft, void *fa) -{ - FP_DECL_D(A); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - FP_CONV(S, D, 1, 2, R, A); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fd2si.c b/arch/nds32/math-emu/fd2si.c deleted file mode 100644 index fae3e16a0a10..000000000000 --- a/arch/nds32/math-emu/fd2si.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fd2si(void *ft, void *fa) -{ - int r; - - FP_DECL_D(A); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - if (A_c == FP_CLS_INF) { - *(int *)ft = (A_s == 0) ? 0x7fffffff : 0x80000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_ROUND_D(r, A, 32, 1); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(int *)ft = r; - } - -} diff --git a/arch/nds32/math-emu/fd2siz.c b/arch/nds32/math-emu/fd2siz.c deleted file mode 100644 index 92fe6774f112..000000000000 --- a/arch/nds32/math-emu/fd2siz.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fd2si_z(void *ft, void *fa) -{ - int r; - - FP_DECL_D(A); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - if (A_c == FP_CLS_INF) { - *(int *)ft = (A_s == 0) ? 0x7fffffff : 0x80000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_D(r, A, 32, 1); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(int *)ft = r; - } - -} diff --git a/arch/nds32/math-emu/fd2ui.c b/arch/nds32/math-emu/fd2ui.c deleted file mode 100644 index a0423b699aa4..000000000000 --- a/arch/nds32/math-emu/fd2ui.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fd2ui(void *ft, void *fa) -{ - unsigned int r; - - FP_DECL_D(A); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - if (A_c == FP_CLS_INF) { - *(unsigned int *)ft = (A_s == 0) ? 0xffffffff : 0x00000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(unsigned int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_ROUND_D(r, A, 32, 0); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(unsigned int *)ft = r; - } - -} diff --git a/arch/nds32/math-emu/fd2uiz.c b/arch/nds32/math-emu/fd2uiz.c deleted file mode 100644 index 8ae17cfce90d..000000000000 --- a/arch/nds32/math-emu/fd2uiz.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fd2ui_z(void *ft, void *fa) -{ - unsigned int r; - - FP_DECL_D(A); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - if (A_c == FP_CLS_INF) { - *(unsigned int *)ft = (A_s == 0) ? 0xffffffff : 0x00000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(unsigned int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_D(r, A, 32, 0); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(unsigned int *)ft = r; - } - -} diff --git a/arch/nds32/math-emu/fdivd.c b/arch/nds32/math-emu/fdivd.c deleted file mode 100644 index 458e7e98b08e..000000000000 --- a/arch/nds32/math-emu/fdivd.c +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <linux/uaccess.h> -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fdivd(void *ft, void *fa, void *fb) -{ - FP_DECL_D(A); - FP_DECL_D(B); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - FP_UNPACK_DP(B, fb); - - if (B_c == FP_CLS_ZERO && A_c != FP_CLS_ZERO) - FP_SET_EXCEPTION(FP_EX_DIVZERO); - - FP_DIV_D(R, A, B); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fdivs.c b/arch/nds32/math-emu/fdivs.c deleted file mode 100644 index c7d202159ce2..000000000000 --- a/arch/nds32/math-emu/fdivs.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fdivs(void *ft, void *fa, void *fb) -{ - FP_DECL_S(A); - FP_DECL_S(B); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - FP_UNPACK_SP(B, fb); - - if (B_c == FP_CLS_ZERO && A_c != FP_CLS_ZERO) - FP_SET_EXCEPTION(FP_EX_DIVZERO); - - FP_DIV_S(R, A, B); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fmuld.c b/arch/nds32/math-emu/fmuld.c deleted file mode 100644 index f3c77a45ddc2..000000000000 --- a/arch/nds32/math-emu/fmuld.c +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -void fmuld(void *ft, void *fa, void *fb) -{ - FP_DECL_D(A); - FP_DECL_D(B); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - FP_UNPACK_DP(B, fb); - - FP_MUL_D(R, A, B); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fmuls.c b/arch/nds32/math-emu/fmuls.c deleted file mode 100644 index cf150df938f9..000000000000 --- a/arch/nds32/math-emu/fmuls.c +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fmuls(void *ft, void *fa, void *fb) -{ - FP_DECL_S(A); - FP_DECL_S(B); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - FP_UNPACK_SP(B, fb); - - FP_MUL_S(R, A, B); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fnegd.c b/arch/nds32/math-emu/fnegd.c deleted file mode 100644 index de7ea6a0873e..000000000000 --- a/arch/nds32/math-emu/fnegd.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -void fnegd(void *ft, void *fa) -{ - FP_DECL_D(A); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - FP_NEG_D(R, A); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fnegs.c b/arch/nds32/math-emu/fnegs.c deleted file mode 100644 index 07270b326a77..000000000000 --- a/arch/nds32/math-emu/fnegs.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fnegs(void *ft, void *fa) -{ - FP_DECL_S(A); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - FP_NEG_S(R, A); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fpuemu.c b/arch/nds32/math-emu/fpuemu.c deleted file mode 100644 index 46558a15c0dc..000000000000 --- a/arch/nds32/math-emu/fpuemu.c +++ /dev/null @@ -1,406 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <asm/bitfield.h> -#include <asm/uaccess.h> -#include <asm/sfp-machine.h> -#include <asm/fpuemu.h> -#include <asm/nds32_fpu_inst.h> - -#define DPFROMREG(dp, x) (dp = (void *)((unsigned long *)fpu_reg + 2*x)) -#ifdef __NDS32_EL__ -#define SPFROMREG(sp, x)\ - ((sp) = (void *)((unsigned long *)fpu_reg + (x^1))) -#else -#define SPFROMREG(sp, x) ((sp) = (void *)((unsigned long *)fpu_reg + x)) -#endif - -#define DEF3OP(name, p, f1, f2) \ -void fpemu_##name##p(void *ft, void *fa, void *fb) \ -{ \ - f1(fa, fa, fb); \ - f2(ft, ft, fa); \ -} - -#define DEF3OPNEG(name, p, f1, f2, f3) \ -void fpemu_##name##p(void *ft, void *fa, void *fb) \ -{ \ - f1(fa, fa, fb); \ - f2(ft, ft, fa); \ - f3(ft, ft); \ -} -DEF3OP(fmadd, s, fmuls, fadds); -DEF3OP(fmsub, s, fmuls, fsubs); -DEF3OP(fmadd, d, fmuld, faddd); -DEF3OP(fmsub, d, fmuld, fsubd); -DEF3OPNEG(fnmadd, s, fmuls, fadds, fnegs); -DEF3OPNEG(fnmsub, s, fmuls, fsubs, fnegs); -DEF3OPNEG(fnmadd, d, fmuld, faddd, fnegd); -DEF3OPNEG(fnmsub, d, fmuld, fsubd, fnegd); - -static const unsigned char cmptab[8] = { - SF_CEQ, - SF_CEQ, - SF_CLT, - SF_CLT, - SF_CLT | SF_CEQ, - SF_CLT | SF_CEQ, - SF_CUN, - SF_CUN -}; - -enum ARGTYPE { - S1S = 1, - S2S, - S1D, - CS, - D1D, - D2D, - D1S, - CD -}; -union func_t { - void (*t)(void *ft, void *fa, void *fb); - void (*b)(void *ft, void *fa); -}; -/* - * Emulate a single FPU arithmetic instruction. - */ -static int fpu_emu(struct fpu_struct *fpu_reg, unsigned long insn) -{ - int rfmt; /* resulting format */ - union func_t func; - int ftype = 0; - - switch (rfmt = NDS32Insn_OPCODE_COP0(insn)) { - case fs1_op:{ - switch (NDS32Insn_OPCODE_BIT69(insn)) { - case fadds_op: - func.t = fadds; - ftype = S2S; - break; - case fsubs_op: - func.t = fsubs; - ftype = S2S; - break; - case fmadds_op: - func.t = fpemu_fmadds; - ftype = S2S; - break; - case fmsubs_op: - func.t = fpemu_fmsubs; - ftype = S2S; - break; - case fnmadds_op: - func.t = fpemu_fnmadds; - ftype = S2S; - break; - case fnmsubs_op: - func.t = fpemu_fnmsubs; - ftype = S2S; - break; - case fmuls_op: - func.t = fmuls; - ftype = S2S; - break; - case fdivs_op: - func.t = fdivs; - ftype = S2S; - break; - case fs1_f2op_op: - switch (NDS32Insn_OPCODE_BIT1014(insn)) { - case fs2d_op: - func.b = fs2d; - ftype = S1D; - break; - case fs2si_op: - func.b = fs2si; - ftype = S1S; - break; - case fs2si_z_op: - func.b = fs2si_z; - ftype = S1S; - break; - case fs2ui_op: - func.b = fs2ui; - ftype = S1S; - break; - case fs2ui_z_op: - func.b = fs2ui_z; - ftype = S1S; - break; - case fsi2s_op: - func.b = fsi2s; - ftype = S1S; - break; - case fui2s_op: - func.b = fui2s; - ftype = S1S; - break; - case fsqrts_op: - func.b = fsqrts; - ftype = S1S; - break; - default: - return SIGILL; - } - break; - default: - return SIGILL; - } - break; - } - case fs2_op: - switch (NDS32Insn_OPCODE_BIT69(insn)) { - case fcmpeqs_op: - case fcmpeqs_e_op: - case fcmplts_op: - case fcmplts_e_op: - case fcmples_op: - case fcmples_e_op: - case fcmpuns_op: - case fcmpuns_e_op: - ftype = CS; - break; - default: - return SIGILL; - } - break; - case fd1_op:{ - switch (NDS32Insn_OPCODE_BIT69(insn)) { - case faddd_op: - func.t = faddd; - ftype = D2D; - break; - case fsubd_op: - func.t = fsubd; - ftype = D2D; - break; - case fmaddd_op: - func.t = fpemu_fmaddd; - ftype = D2D; - break; - case fmsubd_op: - func.t = fpemu_fmsubd; - ftype = D2D; - break; - case fnmaddd_op: - func.t = fpemu_fnmaddd; - ftype = D2D; - break; - case fnmsubd_op: - func.t = fpemu_fnmsubd; - ftype = D2D; - break; - case fmuld_op: - func.t = fmuld; - ftype = D2D; - break; - case fdivd_op: - func.t = fdivd; - ftype = D2D; - break; - case fd1_f2op_op: - switch (NDS32Insn_OPCODE_BIT1014(insn)) { - case fd2s_op: - func.b = fd2s; - ftype = D1S; - break; - case fd2si_op: - func.b = fd2si; - ftype = D1S; - break; - case fd2si_z_op: - func.b = fd2si_z; - ftype = D1S; - break; - case fd2ui_op: - func.b = fd2ui; - ftype = D1S; - break; - case fd2ui_z_op: - func.b = fd2ui_z; - ftype = D1S; - break; - case fsi2d_op: - func.b = fsi2d; - ftype = D1S; - break; - case fui2d_op: - func.b = fui2d; - ftype = D1S; - break; - case fsqrtd_op: - func.b = fsqrtd; - ftype = D1D; - break; - default: - return SIGILL; - } - break; - default: - return SIGILL; - - } - break; - } - - case fd2_op: - switch (NDS32Insn_OPCODE_BIT69(insn)) { - case fcmpeqd_op: - case fcmpeqd_e_op: - case fcmpltd_op: - case fcmpltd_e_op: - case fcmpled_op: - case fcmpled_e_op: - case fcmpund_op: - case fcmpund_e_op: - ftype = CD; - break; - default: - return SIGILL; - } - break; - - default: - return SIGILL; - } - - switch (ftype) { - case S1S:{ - void *ft, *fa; - - SPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - SPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - func.b(ft, fa); - break; - } - case S2S:{ - void *ft, *fa, *fb; - - SPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - SPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - SPFROMREG(fb, NDS32Insn_OPCODE_Rb(insn)); - func.t(ft, fa, fb); - break; - } - case S1D:{ - void *ft, *fa; - - DPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - SPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - func.b(ft, fa); - break; - } - case CS:{ - unsigned int cmpop = NDS32Insn_OPCODE_BIT69(insn); - void *ft, *fa, *fb; - - SPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - SPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - SPFROMREG(fb, NDS32Insn_OPCODE_Rb(insn)); - if (cmpop < 0x8) { - cmpop = cmptab[cmpop]; - fcmps(ft, fa, fb, cmpop); - } else - return SIGILL; - break; - } - case D1D:{ - void *ft, *fa; - - DPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - DPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - func.b(ft, fa); - break; - } - case D2D:{ - void *ft, *fa, *fb; - - DPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - DPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - DPFROMREG(fb, NDS32Insn_OPCODE_Rb(insn)); - func.t(ft, fa, fb); - break; - } - case D1S:{ - void *ft, *fa; - - SPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - DPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - func.b(ft, fa); - break; - } - case CD:{ - unsigned int cmpop = NDS32Insn_OPCODE_BIT69(insn); - void *ft, *fa, *fb; - - SPFROMREG(ft, NDS32Insn_OPCODE_Rt(insn)); - DPFROMREG(fa, NDS32Insn_OPCODE_Ra(insn)); - DPFROMREG(fb, NDS32Insn_OPCODE_Rb(insn)); - if (cmpop < 0x8) { - cmpop = cmptab[cmpop]; - fcmpd(ft, fa, fb, cmpop); - } else - return SIGILL; - break; - } - default: - return SIGILL; - } - - /* - * If an exception is required, generate a tidy SIGFPE exception. - */ -#if IS_ENABLED(CONFIG_SUPPORT_DENORMAL_ARITHMETIC) - if (((fpu_reg->fpcsr << 5) & fpu_reg->fpcsr & FPCSR_mskALLE_NO_UDF_IEXE) - || ((fpu_reg->fpcsr << 5) & (fpu_reg->UDF_IEX_trap))) { -#else - if ((fpu_reg->fpcsr << 5) & fpu_reg->fpcsr & FPCSR_mskALLE) { -#endif - return SIGFPE; - } - return 0; -} - -int do_fpuemu(struct pt_regs *regs, struct fpu_struct *fpu) -{ - unsigned long insn = 0, addr = regs->ipc; - unsigned long emulpc, contpc; - unsigned char *pc = (void *)&insn; - char c; - int i = 0, ret; - - for (i = 0; i < 4; i++) { - if (__get_user(c, (unsigned char *)addr++)) - return SIGBUS; - *pc++ = c; - } - - insn = be32_to_cpu(insn); - - emulpc = regs->ipc; - contpc = regs->ipc + 4; - - if (NDS32Insn_OPCODE(insn) != cop0_op) - return SIGILL; - - switch (NDS32Insn_OPCODE_COP0(insn)) { - case fs1_op: - case fs2_op: - case fd1_op: - case fd2_op: - { - /* a real fpu computation instruction */ - ret = fpu_emu(fpu, insn); - if (!ret) - regs->ipc = contpc; - } - break; - - default: - return SIGILL; - } - - return ret; -} diff --git a/arch/nds32/math-emu/fs2d.c b/arch/nds32/math-emu/fs2d.c deleted file mode 100644 index 0e8db9035631..000000000000 --- a/arch/nds32/math-emu/fs2d.c +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <linux/uaccess.h> -#include <asm/sfp-machine.h> -#include <math-emu/double.h> -#include <math-emu/single.h> -#include <math-emu/soft-fp.h> - -void fs2d(void *ft, void *fa) -{ - FP_DECL_S(A); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - FP_CONV(D, S, 2, 1, R, A); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fs2si.c b/arch/nds32/math-emu/fs2si.c deleted file mode 100644 index b4931d60980e..000000000000 --- a/arch/nds32/math-emu/fs2si.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fs2si(void *ft, void *fa) -{ - int r; - - FP_DECL_S(A); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - if (A_c == FP_CLS_INF) { - *(int *)ft = (A_s == 0) ? 0x7fffffff : 0x80000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_ROUND_S(r, A, 32, 1); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(int *)ft = r; - } -} diff --git a/arch/nds32/math-emu/fs2siz.c b/arch/nds32/math-emu/fs2siz.c deleted file mode 100644 index 1c2b99ce3e38..000000000000 --- a/arch/nds32/math-emu/fs2siz.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fs2si_z(void *ft, void *fa) -{ - int r; - - FP_DECL_S(A); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - if (A_c == FP_CLS_INF) { - *(int *)ft = (A_s == 0) ? 0x7fffffff : 0x80000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_S(r, A, 32, 1); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(int *)ft = r; - } -} diff --git a/arch/nds32/math-emu/fs2ui.c b/arch/nds32/math-emu/fs2ui.c deleted file mode 100644 index c337f0384d06..000000000000 --- a/arch/nds32/math-emu/fs2ui.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fs2ui(void *ft, void *fa) -{ - unsigned int r; - - FP_DECL_S(A); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - if (A_c == FP_CLS_INF) { - *(unsigned int *)ft = (A_s == 0) ? 0xffffffff : 0x00000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(unsigned int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_ROUND_S(r, A, 32, 0); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(unsigned int *)ft = r; - } -} diff --git a/arch/nds32/math-emu/fs2uiz.c b/arch/nds32/math-emu/fs2uiz.c deleted file mode 100644 index 22c5e4768044..000000000000 --- a/arch/nds32/math-emu/fs2uiz.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fs2ui_z(void *ft, void *fa) -{ - unsigned int r; - - FP_DECL_S(A); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - if (A_c == FP_CLS_INF) { - *(unsigned int *)ft = (A_s == 0) ? 0xffffffff : 0x00000000; - __FPU_FPCSR |= FP_EX_INVALID; - } else if (A_c == FP_CLS_NAN) { - *(unsigned int *)ft = 0xffffffff; - __FPU_FPCSR |= FP_EX_INVALID; - } else { - FP_TO_INT_S(r, A, 32, 0); - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - *(unsigned int *)ft = r; - } - -} diff --git a/arch/nds32/math-emu/fsi2d.c b/arch/nds32/math-emu/fsi2d.c deleted file mode 100644 index 6b04cec0c5c5..000000000000 --- a/arch/nds32/math-emu/fsi2d.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fsi2d(void *ft, void *fa) -{ - int a = *(int *)fa; - - FP_DECL_D(R); - FP_DECL_EX; - - FP_FROM_INT_D(R, a, 32, int); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/math-emu/fsi2s.c b/arch/nds32/math-emu/fsi2s.c deleted file mode 100644 index 689864a5df90..000000000000 --- a/arch/nds32/math-emu/fsi2s.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fsi2s(void *ft, void *fa) -{ - int a = *(int *)fa; - - FP_DECL_S(R); - FP_DECL_EX; - - FP_FROM_INT_S(R, a, 32, int); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/math-emu/fsqrtd.c b/arch/nds32/math-emu/fsqrtd.c deleted file mode 100644 index c3a8dbd81d4e..000000000000 --- a/arch/nds32/math-emu/fsqrtd.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <linux/uaccess.h> -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -void fsqrtd(void *ft, void *fa) -{ - FP_DECL_D(A); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - - FP_SQRT_D(R, A); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fsqrts.c b/arch/nds32/math-emu/fsqrts.c deleted file mode 100644 index 4c6f94b27328..000000000000 --- a/arch/nds32/math-emu/fsqrts.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation - -#include <linux/uaccess.h> -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fsqrts(void *ft, void *fa) -{ - FP_DECL_S(A); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - - FP_SQRT_S(R, A); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fsubd.c b/arch/nds32/math-emu/fsubd.c deleted file mode 100644 index 81b6a0d02a1f..000000000000 --- a/arch/nds32/math-emu/fsubd.c +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> -void fsubd(void *ft, void *fa, void *fb) -{ - - FP_DECL_D(A); - FP_DECL_D(B); - FP_DECL_D(R); - FP_DECL_EX; - - FP_UNPACK_DP(A, fa); - FP_UNPACK_DP(B, fb); - - if (B_c != FP_CLS_NAN) - B_s ^= 1; - - FP_ADD_D(R, A, B); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fsubs.c b/arch/nds32/math-emu/fsubs.c deleted file mode 100644 index 61ddd9708465..000000000000 --- a/arch/nds32/math-emu/fsubs.c +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2018 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -void fsubs(void *ft, void *fa, void *fb) -{ - - FP_DECL_S(A); - FP_DECL_S(B); - FP_DECL_S(R); - FP_DECL_EX; - - FP_UNPACK_SP(A, fa); - FP_UNPACK_SP(B, fb); - - if (B_c != FP_CLS_NAN) - B_s ^= 1; - - FP_ADD_S(R, A, B); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; -} diff --git a/arch/nds32/math-emu/fui2d.c b/arch/nds32/math-emu/fui2d.c deleted file mode 100644 index 9689d33a8d50..000000000000 --- a/arch/nds32/math-emu/fui2d.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/double.h> - -void fui2d(void *ft, void *fa) -{ - unsigned int a = *(unsigned int *)fa; - - FP_DECL_D(R); - FP_DECL_EX; - - FP_FROM_INT_D(R, a, 32, int); - - FP_PACK_DP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/math-emu/fui2s.c b/arch/nds32/math-emu/fui2s.c deleted file mode 100644 index f70f0762547d..000000000000 --- a/arch/nds32/math-emu/fui2s.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2019 Andes Technology Corporation -#include <linux/uaccess.h> - -#include <asm/sfp-machine.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> - -void fui2s(void *ft, void *fa) -{ - unsigned int a = *(unsigned int *)fa; - - FP_DECL_S(R); - FP_DECL_EX; - - FP_FROM_INT_S(R, a, 32, int); - - FP_PACK_SP(ft, R); - - __FPU_FPCSR |= FP_CUR_EXCEPTIONS; - -} diff --git a/arch/nds32/mm/Makefile b/arch/nds32/mm/Makefile deleted file mode 100644 index 14fb2e8eb036..000000000000 --- a/arch/nds32/mm/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y := extable.o tlb.o fault.o init.o mmap.o \ - mm-nds32.o cacheflush.o proc.o - -obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o - -ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_proc.o = $(CC_FLAGS_FTRACE) -endif -CFLAGS_proc.o += -fomit-frame-pointer diff --git a/arch/nds32/mm/alignment.c b/arch/nds32/mm/alignment.c deleted file mode 100644 index 1eb7ded6992b..000000000000 --- a/arch/nds32/mm/alignment.c +++ /dev/null @@ -1,578 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/proc_fs.h> -#include <linux/uaccess.h> -#include <linux/sysctl.h> -#include <asm/unaligned.h> - -#define DEBUG(enable, tagged, ...) \ - do{ \ - if (enable) { \ - if (tagged) \ - pr_warn("[ %30s() ] ", __func__); \ - pr_warn(__VA_ARGS__); \ - } \ - } while (0) - -#define RT(inst) (((inst) >> 20) & 0x1FUL) -#define RA(inst) (((inst) >> 15) & 0x1FUL) -#define RB(inst) (((inst) >> 10) & 0x1FUL) -#define SV(inst) (((inst) >> 8) & 0x3UL) -#define IMM(inst) (((inst) >> 0) & 0x7FFFUL) - -#define RA3(inst) (((inst) >> 3) & 0x7UL) -#define RT3(inst) (((inst) >> 6) & 0x7UL) -#define IMM3U(inst) (((inst) >> 0) & 0x7UL) - -#define RA5(inst) (((inst) >> 0) & 0x1FUL) -#define RT4(inst) (((inst) >> 5) & 0xFUL) - -#define GET_IMMSVAL(imm_value) \ - (((imm_value >> 14) & 0x1) ? (imm_value - 0x8000) : imm_value) - -#define __get8_data(val,addr,err) \ - __asm__( \ - "1: lbi.bi %1, [%2], #1\n" \ - "2:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: movi %0, #1\n" \ - " j 2b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .popsection\n" \ - : "=r" (err), "=&r" (val), "=r" (addr) \ - : "0" (err), "2" (addr)) - -#define get16_data(addr, val_ptr) \ - do { \ - unsigned int err = 0, v, a = addr; \ - __get8_data(v,a,err); \ - *val_ptr = v << 0; \ - __get8_data(v,a,err); \ - *val_ptr |= v << 8; \ - if (err) \ - goto fault; \ - *val_ptr = le16_to_cpu(*val_ptr); \ - } while(0) - -#define get32_data(addr, val_ptr) \ - do { \ - unsigned int err = 0, v, a = addr; \ - __get8_data(v,a,err); \ - *val_ptr = v << 0; \ - __get8_data(v,a,err); \ - *val_ptr |= v << 8; \ - __get8_data(v,a,err); \ - *val_ptr |= v << 16; \ - __get8_data(v,a,err); \ - *val_ptr |= v << 24; \ - if (err) \ - goto fault; \ - *val_ptr = le32_to_cpu(*val_ptr); \ - } while(0) - -#define get_data(addr, val_ptr, len) \ - if (len == 2) \ - get16_data(addr, val_ptr); \ - else \ - get32_data(addr, val_ptr); - -#define set16_data(addr, val) \ - do { \ - unsigned int err = 0, *ptr = addr ; \ - val = le32_to_cpu(val); \ - __asm__( \ - "1: sbi.bi %2, [%1], #1\n" \ - " srli %2, %2, #8\n" \ - "2: sbi %2, [%1]\n" \ - "3:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "4: movi %0, #1\n" \ - " j 3b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 4b\n" \ - " .long 2b, 4b\n" \ - " .popsection\n" \ - : "=r" (err), "+r" (ptr), "+r" (val) \ - : "0" (err) \ - ); \ - if (err) \ - goto fault; \ - } while(0) - -#define set32_data(addr, val) \ - do { \ - unsigned int err = 0, *ptr = addr ; \ - val = le32_to_cpu(val); \ - __asm__( \ - "1: sbi.bi %2, [%1], #1\n" \ - " srli %2, %2, #8\n" \ - "2: sbi.bi %2, [%1], #1\n" \ - " srli %2, %2, #8\n" \ - "3: sbi.bi %2, [%1], #1\n" \ - " srli %2, %2, #8\n" \ - "4: sbi %2, [%1]\n" \ - "5:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "6: movi %0, #1\n" \ - " j 5b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 6b\n" \ - " .long 2b, 6b\n" \ - " .long 3b, 6b\n" \ - " .long 4b, 6b\n" \ - " .popsection\n" \ - : "=r" (err), "+r" (ptr), "+r" (val) \ - : "0" (err) \ - ); \ - if (err) \ - goto fault; \ - } while(0) -#define set_data(addr, val, len) \ - if (len == 2) \ - set16_data(addr, val); \ - else \ - set32_data(addr, val); -#define NDS32_16BIT_INSTRUCTION 0x80000000 - -extern pte_t va_present(struct mm_struct *mm, unsigned long addr); -extern pte_t va_kernel_present(unsigned long addr); -extern int va_readable(struct pt_regs *regs, unsigned long addr); -extern int va_writable(struct pt_regs *regs, unsigned long addr); - -int unalign_access_mode = 0, unalign_access_debug = 0; - -static inline unsigned long *idx_to_addr(struct pt_regs *regs, int idx) -{ - /* this should be consistent with ptrace.h */ - if (idx >= 0 && idx <= 25) /* R0-R25 */ - return ®s->uregs[0] + idx; - else if (idx >= 28 && idx <= 30) /* FP, GP, LP */ - return ®s->fp + (idx - 28); - else if (idx == 31) /* SP */ - return ®s->sp; - else - return NULL; /* cause a segfault */ -} - -static inline unsigned long get_inst(unsigned long addr) -{ - return be32_to_cpu(get_unaligned((u32 *) addr)); -} - -static inline unsigned long sign_extend(unsigned long val, int len) -{ - unsigned long ret = 0; - unsigned char *s, *t; - int i = 0; - - val = cpu_to_le32(val); - - s = (void *)&val; - t = (void *)&ret; - - while (i++ < len) - *t++ = *s++; - - if (((*(t - 1)) & 0x80) && (i < 4)) { - - while (i++ <= 4) - *t++ = 0xff; - } - - return le32_to_cpu(ret); -} - -static inline int do_16(unsigned long inst, struct pt_regs *regs) -{ - int imm, regular, load, len, addr_mode, idx_mode; - unsigned long unaligned_addr, target_val, source_idx, target_idx, - shift = 0; - switch ((inst >> 9) & 0x3F) { - - case 0x12: /* LHI333 */ - imm = 1; - regular = 1; - load = 1; - len = 2; - addr_mode = 3; - idx_mode = 3; - break; - case 0x10: /* LWI333 */ - imm = 1; - regular = 1; - load = 1; - len = 4; - addr_mode = 3; - idx_mode = 3; - break; - case 0x11: /* LWI333.bi */ - imm = 1; - regular = 0; - load = 1; - len = 4; - addr_mode = 3; - idx_mode = 3; - break; - case 0x1A: /* LWI450 */ - imm = 0; - regular = 1; - load = 1; - len = 4; - addr_mode = 5; - idx_mode = 4; - break; - case 0x16: /* SHI333 */ - imm = 1; - regular = 1; - load = 0; - len = 2; - addr_mode = 3; - idx_mode = 3; - break; - case 0x14: /* SWI333 */ - imm = 1; - regular = 1; - load = 0; - len = 4; - addr_mode = 3; - idx_mode = 3; - break; - case 0x15: /* SWI333.bi */ - imm = 1; - regular = 0; - load = 0; - len = 4; - addr_mode = 3; - idx_mode = 3; - break; - case 0x1B: /* SWI450 */ - imm = 0; - regular = 1; - load = 0; - len = 4; - addr_mode = 5; - idx_mode = 4; - break; - - default: - return -EFAULT; - } - - if (addr_mode == 3) { - unaligned_addr = *idx_to_addr(regs, RA3(inst)); - source_idx = RA3(inst); - } else { - unaligned_addr = *idx_to_addr(regs, RA5(inst)); - source_idx = RA5(inst); - } - - if (idx_mode == 3) - target_idx = RT3(inst); - else - target_idx = RT4(inst); - - if (imm) - shift = IMM3U(inst) * len; - - if (regular) - unaligned_addr += shift; - - if (load) { - if (!access_ok((void *)unaligned_addr, len)) - return -EACCES; - - get_data(unaligned_addr, &target_val, len); - *idx_to_addr(regs, target_idx) = target_val; - } else { - if (!access_ok((void *)unaligned_addr, len)) - return -EACCES; - target_val = *idx_to_addr(regs, target_idx); - set_data((void *)unaligned_addr, target_val, len); - } - - if (!regular) - *idx_to_addr(regs, source_idx) = unaligned_addr + shift; - regs->ipc += 2; - - return 0; -fault: - return -EACCES; -} - -static inline int do_32(unsigned long inst, struct pt_regs *regs) -{ - int imm, regular, load, len, sign_ext; - unsigned long unaligned_addr, target_val, shift; - - unaligned_addr = *idx_to_addr(regs, RA(inst)); - - switch ((inst >> 25) << 1) { - - case 0x02: /* LHI */ - imm = 1; - regular = 1; - load = 1; - len = 2; - sign_ext = 0; - break; - case 0x0A: /* LHI.bi */ - imm = 1; - regular = 0; - load = 1; - len = 2; - sign_ext = 0; - break; - case 0x22: /* LHSI */ - imm = 1; - regular = 1; - load = 1; - len = 2; - sign_ext = 1; - break; - case 0x2A: /* LHSI.bi */ - imm = 1; - regular = 0; - load = 1; - len = 2; - sign_ext = 1; - break; - case 0x04: /* LWI */ - imm = 1; - regular = 1; - load = 1; - len = 4; - sign_ext = 0; - break; - case 0x0C: /* LWI.bi */ - imm = 1; - regular = 0; - load = 1; - len = 4; - sign_ext = 0; - break; - case 0x12: /* SHI */ - imm = 1; - regular = 1; - load = 0; - len = 2; - sign_ext = 0; - break; - case 0x1A: /* SHI.bi */ - imm = 1; - regular = 0; - load = 0; - len = 2; - sign_ext = 0; - break; - case 0x14: /* SWI */ - imm = 1; - regular = 1; - load = 0; - len = 4; - sign_ext = 0; - break; - case 0x1C: /* SWI.bi */ - imm = 1; - regular = 0; - load = 0; - len = 4; - sign_ext = 0; - break; - - default: - switch (inst & 0xff) { - - case 0x01: /* LH */ - imm = 0; - regular = 1; - load = 1; - len = 2; - sign_ext = 0; - break; - case 0x05: /* LH.bi */ - imm = 0; - regular = 0; - load = 1; - len = 2; - sign_ext = 0; - break; - case 0x11: /* LHS */ - imm = 0; - regular = 1; - load = 1; - len = 2; - sign_ext = 1; - break; - case 0x15: /* LHS.bi */ - imm = 0; - regular = 0; - load = 1; - len = 2; - sign_ext = 1; - break; - case 0x02: /* LW */ - imm = 0; - regular = 1; - load = 1; - len = 4; - sign_ext = 0; - break; - case 0x06: /* LW.bi */ - imm = 0; - regular = 0; - load = 1; - len = 4; - sign_ext = 0; - break; - case 0x09: /* SH */ - imm = 0; - regular = 1; - load = 0; - len = 2; - sign_ext = 0; - break; - case 0x0D: /* SH.bi */ - imm = 0; - regular = 0; - load = 0; - len = 2; - sign_ext = 0; - break; - case 0x0A: /* SW */ - imm = 0; - regular = 1; - load = 0; - len = 4; - sign_ext = 0; - break; - case 0x0E: /* SW.bi */ - imm = 0; - regular = 0; - load = 0; - len = 4; - sign_ext = 0; - break; - - default: - return -EFAULT; - } - } - - if (imm) - shift = GET_IMMSVAL(IMM(inst)) * len; - else - shift = *idx_to_addr(regs, RB(inst)) << SV(inst); - - if (regular) - unaligned_addr += shift; - - if (load) { - - if (!access_ok((void *)unaligned_addr, len)) - return -EACCES; - - get_data(unaligned_addr, &target_val, len); - - if (sign_ext) - *idx_to_addr(regs, RT(inst)) = - sign_extend(target_val, len); - else - *idx_to_addr(regs, RT(inst)) = target_val; - } else { - - if (!access_ok((void *)unaligned_addr, len)) - return -EACCES; - - target_val = *idx_to_addr(regs, RT(inst)); - set_data((void *)unaligned_addr, target_val, len); - } - - if (!regular) - *idx_to_addr(regs, RA(inst)) = unaligned_addr + shift; - - regs->ipc += 4; - - return 0; -fault: - return -EACCES; -} - -int do_unaligned_access(unsigned long addr, struct pt_regs *regs) -{ - unsigned long inst; - int ret = -EFAULT; - mm_segment_t seg; - - inst = get_inst(regs->ipc); - - DEBUG((unalign_access_debug > 0), 1, - "Faulting addr: 0x%08lx, pc: 0x%08lx [inst: 0x%08lx ]\n", addr, - regs->ipc, inst); - - seg = force_uaccess_begin(); - if (inst & NDS32_16BIT_INSTRUCTION) - ret = do_16((inst >> 16) & 0xffff, regs); - else - ret = do_32(inst, regs); - force_uaccess_end(seg); - - return ret; -} - -#ifdef CONFIG_PROC_FS - -static struct ctl_table alignment_tbl[3] = { - { - .procname = "enable", - .data = &unalign_access_mode, - .maxlen = sizeof(unalign_access_mode), - .mode = 0666, - .proc_handler = &proc_dointvec - } - , - { - .procname = "debug_info", - .data = &unalign_access_debug, - .maxlen = sizeof(unalign_access_debug), - .mode = 0644, - .proc_handler = &proc_dointvec - } - , - {} -}; - -static struct ctl_table nds32_sysctl_table[2] = { - { - .procname = "unaligned_access", - .mode = 0555, - .child = alignment_tbl}, - {} -}; - -static struct ctl_path nds32_path[2] = { - {.procname = "nds32"}, - {} -}; - -/* - * Initialize nds32 alignment-correction interface - */ -static int __init nds32_sysctl_init(void) -{ - register_sysctl_paths(nds32_path, nds32_sysctl_table); - return 0; -} - -__initcall(nds32_sysctl_init); -#endif /* CONFIG_PROC_FS */ diff --git a/arch/nds32/mm/cacheflush.c b/arch/nds32/mm/cacheflush.c deleted file mode 100644 index 07aac65d1cab..000000000000 --- a/arch/nds32/mm/cacheflush.c +++ /dev/null @@ -1,338 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/mm.h> -#include <linux/sched.h> -#include <linux/fs.h> -#include <linux/pagemap.h> -#include <linux/module.h> -#include <asm/cacheflush.h> -#include <asm/proc-fns.h> -#include <asm/shmparam.h> -#include <asm/cache_info.h> - -extern struct cache_info L1_cache_info[2]; - -void flush_icache_range(unsigned long start, unsigned long end) -{ - unsigned long line_size, flags; - line_size = L1_cache_info[DCACHE].line_size; - start = start & ~(line_size - 1); - end = (end + line_size - 1) & ~(line_size - 1); - local_irq_save(flags); - cpu_cache_wbinval_range(start, end, 1); - local_irq_restore(flags); -} -EXPORT_SYMBOL(flush_icache_range); - -void flush_icache_page(struct vm_area_struct *vma, struct page *page) -{ - unsigned long flags; - unsigned long kaddr; - local_irq_save(flags); - kaddr = (unsigned long)kmap_atomic(page); - cpu_cache_wbinval_page(kaddr, vma->vm_flags & VM_EXEC); - kunmap_atomic((void *)kaddr); - local_irq_restore(flags); -} - -void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long addr, int len) -{ - unsigned long kaddr; - kaddr = (unsigned long)kmap_atomic(page) + (addr & ~PAGE_MASK); - flush_icache_range(kaddr, kaddr + len); - kunmap_atomic((void *)kaddr); -} - -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, - pte_t * pte) -{ - struct page *page; - unsigned long pfn = pte_pfn(*pte); - unsigned long flags; - - if (!pfn_valid(pfn)) - return; - - if (vma->vm_mm == current->active_mm) { - local_irq_save(flags); - __nds32__mtsr_dsb(addr, NDS32_SR_TLB_VPN); - __nds32__tlbop_rwr(*pte); - __nds32__isb(); - local_irq_restore(flags); - } - page = pfn_to_page(pfn); - - if ((test_and_clear_bit(PG_dcache_dirty, &page->flags)) || - (vma->vm_flags & VM_EXEC)) { - unsigned long kaddr; - local_irq_save(flags); - kaddr = (unsigned long)kmap_atomic(page); - cpu_cache_wbinval_page(kaddr, vma->vm_flags & VM_EXEC); - kunmap_atomic((void *)kaddr); - local_irq_restore(flags); - } -} -#ifdef CONFIG_CPU_CACHE_ALIASING -extern pte_t va_present(struct mm_struct *mm, unsigned long addr); - -static inline unsigned long aliasing(unsigned long addr, unsigned long page) -{ - return ((addr & PAGE_MASK) ^ page) & (SHMLBA - 1); -} - -static inline unsigned long kremap0(unsigned long uaddr, unsigned long pa) -{ - unsigned long kaddr, pte; - -#define BASE_ADDR0 0xffffc000 - kaddr = BASE_ADDR0 | (uaddr & L1_cache_info[DCACHE].aliasing_mask); - pte = (pa | PAGE_KERNEL); - __nds32__mtsr_dsb(kaddr, NDS32_SR_TLB_VPN); - __nds32__tlbop_rwlk(pte); - __nds32__isb(); - return kaddr; -} - -static inline void kunmap01(unsigned long kaddr) -{ - __nds32__tlbop_unlk(kaddr); - __nds32__tlbop_inv(kaddr); - __nds32__isb(); -} - -static inline unsigned long kremap1(unsigned long uaddr, unsigned long pa) -{ - unsigned long kaddr, pte; - -#define BASE_ADDR1 0xffff8000 - kaddr = BASE_ADDR1 | (uaddr & L1_cache_info[DCACHE].aliasing_mask); - pte = (pa | PAGE_KERNEL); - __nds32__mtsr_dsb(kaddr, NDS32_SR_TLB_VPN); - __nds32__tlbop_rwlk(pte); - __nds32__isb(); - return kaddr; -} - -void flush_cache_mm(struct mm_struct *mm) -{ - unsigned long flags; - - local_irq_save(flags); - cpu_dcache_wbinval_all(); - cpu_icache_inval_all(); - local_irq_restore(flags); -} - -void flush_cache_dup_mm(struct mm_struct *mm) -{ -} - -void flush_cache_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - unsigned long flags; - - if ((end - start) > 8 * PAGE_SIZE) { - cpu_dcache_wbinval_all(); - if (vma->vm_flags & VM_EXEC) - cpu_icache_inval_all(); - return; - } - local_irq_save(flags); - while (start < end) { - if (va_present(vma->vm_mm, start)) - cpu_cache_wbinval_page(start, vma->vm_flags & VM_EXEC); - start += PAGE_SIZE; - } - local_irq_restore(flags); - return; -} - -void flush_cache_page(struct vm_area_struct *vma, - unsigned long addr, unsigned long pfn) -{ - unsigned long vto, flags; - - local_irq_save(flags); - vto = kremap0(addr, pfn << PAGE_SHIFT); - cpu_cache_wbinval_page(vto, vma->vm_flags & VM_EXEC); - kunmap01(vto); - local_irq_restore(flags); -} - -void flush_cache_vmap(unsigned long start, unsigned long end) -{ - cpu_dcache_wbinval_all(); - cpu_icache_inval_all(); -} - -void flush_cache_vunmap(unsigned long start, unsigned long end) -{ - cpu_dcache_wbinval_all(); - cpu_icache_inval_all(); -} - -void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, - struct page *to) -{ - cpu_dcache_wbinval_page((unsigned long)vaddr); - cpu_icache_inval_page((unsigned long)vaddr); - copy_page(vto, vfrom); - cpu_dcache_wbinval_page((unsigned long)vto); - cpu_icache_inval_page((unsigned long)vto); -} - -void clear_user_page(void *addr, unsigned long vaddr, struct page *page) -{ - cpu_dcache_wbinval_page((unsigned long)vaddr); - cpu_icache_inval_page((unsigned long)vaddr); - clear_page(addr); - cpu_dcache_wbinval_page((unsigned long)addr); - cpu_icache_inval_page((unsigned long)addr); -} - -void copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr, struct vm_area_struct *vma) -{ - unsigned long vto, vfrom, flags, kto, kfrom, pfrom, pto; - kto = ((unsigned long)page_address(to) & PAGE_MASK); - kfrom = ((unsigned long)page_address(from) & PAGE_MASK); - pto = page_to_phys(to); - pfrom = page_to_phys(from); - - local_irq_save(flags); - if (aliasing(vaddr, (unsigned long)kfrom)) - cpu_dcache_wb_page((unsigned long)kfrom); - vto = kremap0(vaddr, pto); - vfrom = kremap1(vaddr, pfrom); - copy_page((void *)vto, (void *)vfrom); - kunmap01(vfrom); - kunmap01(vto); - local_irq_restore(flags); -} - -EXPORT_SYMBOL(copy_user_highpage); - -void clear_user_highpage(struct page *page, unsigned long vaddr) -{ - unsigned long vto, flags, kto; - - kto = ((unsigned long)page_address(page) & PAGE_MASK); - - local_irq_save(flags); - if (aliasing(kto, vaddr) && kto != 0) { - cpu_dcache_inval_page(kto); - cpu_icache_inval_page(kto); - } - vto = kremap0(vaddr, page_to_phys(page)); - clear_page((void *)vto); - kunmap01(vto); - local_irq_restore(flags); -} - -EXPORT_SYMBOL(clear_user_highpage); - -void flush_dcache_page(struct page *page) -{ - struct address_space *mapping; - - mapping = page_mapping_file(page); - if (mapping && !mapping_mapped(mapping)) - set_bit(PG_dcache_dirty, &page->flags); - else { - unsigned long kaddr, flags; - - kaddr = (unsigned long)page_address(page); - local_irq_save(flags); - cpu_dcache_wbinval_page(kaddr); - if (mapping) { - unsigned long vaddr, kto; - - vaddr = page->index << PAGE_SHIFT; - if (aliasing(vaddr, kaddr)) { - kto = kremap0(vaddr, page_to_phys(page)); - cpu_dcache_wbinval_page(kto); - kunmap01(kto); - } - } - local_irq_restore(flags); - } -} -EXPORT_SYMBOL(flush_dcache_page); - -void copy_to_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, void *dst, void *src, int len) -{ - unsigned long line_size, start, end, vto, flags; - - local_irq_save(flags); - vto = kremap0(vaddr, page_to_phys(page)); - dst = (void *)(vto | (vaddr & (PAGE_SIZE - 1))); - memcpy(dst, src, len); - if (vma->vm_flags & VM_EXEC) { - line_size = L1_cache_info[DCACHE].line_size; - start = (unsigned long)dst & ~(line_size - 1); - end = - ((unsigned long)dst + len + line_size - 1) & ~(line_size - - 1); - cpu_cache_wbinval_range(start, end, 1); - } - kunmap01(vto); - local_irq_restore(flags); -} - -void copy_from_user_page(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, void *dst, void *src, int len) -{ - unsigned long vto, flags; - - local_irq_save(flags); - vto = kremap0(vaddr, page_to_phys(page)); - src = (void *)(vto | (vaddr & (PAGE_SIZE - 1))); - memcpy(dst, src, len); - kunmap01(vto); - local_irq_restore(flags); -} - -void flush_anon_page(struct vm_area_struct *vma, - struct page *page, unsigned long vaddr) -{ - unsigned long kaddr, flags, ktmp; - if (!PageAnon(page)) - return; - - if (vma->vm_mm != current->active_mm) - return; - - local_irq_save(flags); - if (vma->vm_flags & VM_EXEC) - cpu_icache_inval_page(vaddr & PAGE_MASK); - kaddr = (unsigned long)page_address(page); - if (aliasing(vaddr, kaddr)) { - ktmp = kremap0(vaddr, page_to_phys(page)); - cpu_dcache_wbinval_page(ktmp); - kunmap01(ktmp); - } - local_irq_restore(flags); -} - -void flush_kernel_vmap_range(void *addr, int size) -{ - unsigned long flags; - local_irq_save(flags); - cpu_dcache_wb_range((unsigned long)addr, (unsigned long)addr + size); - local_irq_restore(flags); -} -EXPORT_SYMBOL(flush_kernel_vmap_range); - -void invalidate_kernel_vmap_range(void *addr, int size) -{ - unsigned long flags; - local_irq_save(flags); - cpu_dcache_inval_range((unsigned long)addr, (unsigned long)addr + size); - local_irq_restore(flags); -} -EXPORT_SYMBOL(invalidate_kernel_vmap_range); -#endif diff --git a/arch/nds32/mm/extable.c b/arch/nds32/mm/extable.c deleted file mode 100644 index db7f0a7c8966..000000000000 --- a/arch/nds32/mm/extable.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/extable.h> -#include <linux/uaccess.h> - -int fixup_exception(struct pt_regs *regs) -{ - const struct exception_table_entry *fixup; - - fixup = search_exception_tables(instruction_pointer(regs)); - if (fixup) - regs->ipc = fixup->fixup; - - return fixup != NULL; -} diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c deleted file mode 100644 index 636977a1c8b9..000000000000 --- a/arch/nds32/mm/fault.c +++ /dev/null @@ -1,396 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/extable.h> -#include <linux/module.h> -#include <linux/signal.h> -#include <linux/ptrace.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <linux/hardirq.h> -#include <linux/uaccess.h> -#include <linux/perf_event.h> - -#include <asm/tlbflush.h> - -extern void __noreturn die(const char *str, struct pt_regs *regs, long err); - -/* - * This is useful to dump out the page tables associated with - * 'addr' in mm 'mm'. - */ -void show_pte(struct mm_struct *mm, unsigned long addr) -{ - pgd_t *pgd; - if (!mm) - mm = &init_mm; - - pr_alert("pgd = %p\n", mm->pgd); - pgd = pgd_offset(mm, addr); - pr_alert("[%08lx] *pgd=%08lx", addr, pgd_val(*pgd)); - - do { - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - if (pgd_none(*pgd)) - break; - - if (pgd_bad(*pgd)) { - pr_alert("(bad)"); - break; - } - - p4d = p4d_offset(pgd, addr); - pud = pud_offset(p4d, addr); - pmd = pmd_offset(pud, addr); -#if PTRS_PER_PMD != 1 - pr_alert(", *pmd=%08lx", pmd_val(*pmd)); -#endif - - if (pmd_none(*pmd)) - break; - - if (pmd_bad(*pmd)) { - pr_alert("(bad)"); - break; - } - - if (IS_ENABLED(CONFIG_HIGHMEM)) - { - pte_t *pte; - /* We must not map this if we have highmem enabled */ - pte = pte_offset_map(pmd, addr); - pr_alert(", *pte=%08lx", pte_val(*pte)); - pte_unmap(pte); - } - } while (0); - - pr_alert("\n"); -} - -void do_page_fault(unsigned long entry, unsigned long addr, - unsigned int error_code, struct pt_regs *regs) -{ - struct task_struct *tsk; - struct mm_struct *mm; - struct vm_area_struct *vma; - int si_code; - vm_fault_t fault; - unsigned int mask = VM_ACCESS_FLAGS; - unsigned int flags = FAULT_FLAG_DEFAULT; - - error_code = error_code & (ITYPE_mskINST | ITYPE_mskETYPE); - tsk = current; - mm = tsk->mm; - si_code = SEGV_MAPERR; - /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * - * NOTE! We MUST NOT take any locks for this case. We may - * be in an interrupt or a critical region, and should - * only copy the information from the master page table, - * nothing more. - */ - if (addr >= TASK_SIZE) { - if (user_mode(regs)) - goto bad_area_nosemaphore; - - if (addr >= TASK_SIZE && addr < VMALLOC_END - && (entry == ENTRY_PTE_NOT_PRESENT)) - goto vmalloc_fault; - else - goto no_context; - } - - /* Send a signal to the task for handling the unalignment access. */ - if (entry == ENTRY_GENERAL_EXCPETION - && error_code == ETYPE_ALIGNMENT_CHECK) { - if (user_mode(regs)) - goto bad_area_nosemaphore; - else - goto no_context; - } - - /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. - */ - if (unlikely(faulthandler_disabled() || !mm)) - goto no_context; - - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); - - /* - * As per x86, we may deadlock here. However, since the kernel only - * validly references user space from well defined areas of the code, - * we can bug out early if this is from code which shouldn't. - */ - if (unlikely(!mmap_read_trylock(mm))) { - if (!user_mode(regs) && - !search_exception_tables(instruction_pointer(regs))) - goto no_context; -retry: - mmap_read_lock(mm); - } else { - /* - * The above down_read_trylock() might have succeeded in which - * case, we'll have missed the might_sleep() from down_read(). - */ - might_sleep(); - if (IS_ENABLED(CONFIG_DEBUG_VM)) { - if (!user_mode(regs) && - !search_exception_tables(instruction_pointer(regs))) - goto no_context; - } - } - - vma = find_vma(mm, addr); - - if (unlikely(!vma)) - goto bad_area; - - if (vma->vm_start <= addr) - goto good_area; - - if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) - goto bad_area; - - if (unlikely(expand_stack(vma, addr))) - goto bad_area; - - /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ - -good_area: - si_code = SEGV_ACCERR; - - /* first do some preliminary protection checks */ - if (entry == ENTRY_PTE_NOT_PRESENT) { - if (error_code & ITYPE_mskINST) - mask = VM_EXEC; - else { - mask = VM_READ | VM_WRITE; - } - } else if (entry == ENTRY_TLB_MISC) { - switch (error_code & ITYPE_mskETYPE) { - case RD_PROT: - mask = VM_READ; - break; - case WRT_PROT: - mask = VM_WRITE; - flags |= FAULT_FLAG_WRITE; - break; - case NOEXEC: - mask = VM_EXEC; - break; - case PAGE_MODIFY: - mask = VM_WRITE; - flags |= FAULT_FLAG_WRITE; - break; - case ACC_BIT: - BUG(); - default: - break; - } - - } - if (!(vma->vm_flags & mask)) - goto bad_area; - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ - - fault = handle_mm_fault(vma, addr, flags, regs); - - /* - * If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_lock because it - * would already be released in __lock_page_or_retry in mm/filemap.c. - */ - if (fault_signal_pending(fault, regs)) { - if (!user_mode(regs)) - goto no_context; - return; - } - - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - else - goto bad_area; - } - - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; - - /* No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ - goto retry; - } - - mmap_read_unlock(mm); - return; - - /* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ -bad_area: - mmap_read_unlock(mm); - -bad_area_nosemaphore: - - /* User mode accesses just cause a SIGSEGV */ - - if (user_mode(regs)) { - tsk->thread.address = addr; - tsk->thread.error_code = error_code; - tsk->thread.trap_no = entry; - force_sig_fault(SIGSEGV, si_code, (void __user *)addr); - return; - } - -no_context: - - /* Are we prepared to handle this kernel fault? - * - * (The kernel has valid exception-points in the source - * when it acesses user-memory. When it fails in one - * of those points, we find it in a table and do a jump - * to some fixup code that loads an appropriate error - * code) - */ - - { - const struct exception_table_entry *entry; - - if ((entry = - search_exception_tables(instruction_pointer(regs))) != - NULL) { - /* Adjust the instruction pointer in the stackframe */ - instruction_pointer(regs) = entry->fixup; - return; - } - } - - /* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - */ - - bust_spinlocks(1); - pr_alert("Unable to handle kernel %s at virtual address %08lx\n", - (addr < PAGE_SIZE) ? "NULL pointer dereference" : - "paging request", addr); - - show_pte(mm, addr); - die("Oops", regs, error_code); - - /* - * We ran out of memory, or some other thing happened to us that made - * us unable to handle the page fault gracefully. - */ - -out_of_memory: - mmap_read_unlock(mm); - if (!user_mode(regs)) - goto no_context; - pagefault_out_of_memory(); - return; - -do_sigbus: - mmap_read_unlock(mm); - - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; - - /* - * Send a sigbus - */ - tsk->thread.address = addr; - tsk->thread.error_code = error_code; - tsk->thread.trap_no = entry; - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)addr); - - return; - -vmalloc_fault: - { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Use current_pgd instead of tsk->active_mm->pgd - * since the latter might be unavailable if this - * code is executed in a misfortunately run irq - * (like inside schedule() between switch_mm and - * switch_to...). - */ - - unsigned int index = pgd_index(addr); - pgd_t *pgd, *pgd_k; - p4d_t *p4d, *p4d_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - pte_t *pte_k; - - pgd = (pgd_t *) __va(__nds32__mfsr(NDS32_SR_L1_PPTB)) + index; - pgd_k = init_mm.pgd + index; - - if (!pgd_present(*pgd_k)) - goto no_context; - - p4d = p4d_offset(pgd, addr); - p4d_k = p4d_offset(pgd_k, addr); - if (!p4d_present(*p4d_k)) - goto no_context; - - pud = pud_offset(p4d, addr); - pud_k = pud_offset(p4d_k, addr); - if (!pud_present(*pud_k)) - goto no_context; - - pmd = pmd_offset(pud, addr); - pmd_k = pmd_offset(pud_k, addr); - if (!pmd_present(*pmd_k)) - goto no_context; - - if (!pmd_present(*pmd)) - set_pmd(pmd, *pmd_k); - else - BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k)); - - /* - * Since the vmalloc area is global, we don't - * need to copy individual PTE's, it is enough to - * copy the pgd pointer into the pte page of the - * root task. If that is there, we'll find our pte if - * it exists. - */ - - /* Make sure the actual PTE exists as well to - * catch kernel vmalloc-area accesses to non-mapped - * addres. If we don't do this, this will just - * silently loop forever. - */ - - pte_k = pte_offset_kernel(pmd_k, addr); - if (!pte_present(*pte_k)) - goto no_context; - - return; - } -} diff --git a/arch/nds32/mm/init.c b/arch/nds32/mm/init.c deleted file mode 100644 index f63f839738c4..000000000000 --- a/arch/nds32/mm/init.c +++ /dev/null @@ -1,263 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 1995-2005 Russell King -// Copyright (C) 2012 ARM Ltd. -// Copyright (C) 2013-2017 Andes Technology Corporation - -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/swap.h> -#include <linux/init.h> -#include <linux/memblock.h> -#include <linux/mman.h> -#include <linux/nodemask.h> -#include <linux/initrd.h> -#include <linux/highmem.h> - -#include <asm/sections.h> -#include <asm/setup.h> -#include <asm/tlb.h> -#include <asm/page.h> - -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); -DEFINE_SPINLOCK(anon_alias_lock); -extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; - -/* - * empty_zero_page is a special page that is used for - * zero-initialized data and COW. - */ -struct page *empty_zero_page; -EXPORT_SYMBOL(empty_zero_page); - -static void __init zone_sizes_init(void) -{ - unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; - - max_zone_pfn[ZONE_NORMAL] = max_low_pfn; -#ifdef CONFIG_HIGHMEM - max_zone_pfn[ZONE_HIGHMEM] = max_pfn; -#endif - free_area_init(max_zone_pfn); - -} - -/* - * Map all physical memory under high_memory into kernel's address space. - * - * This is explicitly coded for two-level page tables, so if you need - * something else then this needs to change. - */ -static void __init map_ram(void) -{ - unsigned long v, p, e; - pgd_t *pge; - p4d_t *p4e; - pud_t *pue; - pmd_t *pme; - pte_t *pte; - /* These mark extents of read-only kernel pages... - * ...from vmlinux.lds.S - */ - - p = (u32) memblock_start_of_DRAM() & PAGE_MASK; - e = min((u32) memblock_end_of_DRAM(), (u32) __pa(high_memory)); - - v = (u32) __va(p); - pge = pgd_offset_k(v); - - while (p < e) { - int j; - p4e = p4d_offset(pge, v); - pue = pud_offset(p4e, v); - pme = pmd_offset(pue, v); - - if ((u32) pue != (u32) pge || (u32) pme != (u32) pge) { - panic("%s: Kernel hardcoded for " - "two-level page tables", __func__); - } - - /* Alloc one page for holding PTE's... */ - pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!pte) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); - set_pmd(pme, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE)); - - /* Fill the newly allocated page with PTE'S */ - for (j = 0; p < e && j < PTRS_PER_PTE; - v += PAGE_SIZE, p += PAGE_SIZE, j++, pte++) { - /* Create mapping between p and v. */ - /* TODO: more fine grant for page access permission */ - set_pte(pte, __pte(p + pgprot_val(PAGE_KERNEL))); - } - - pge++; - } -} -static pmd_t *fixmap_pmd_p; -static void __init fixedrange_init(void) -{ - unsigned long vaddr; - pmd_t *pmd; -#ifdef CONFIG_HIGHMEM - pte_t *pte; -#endif /* CONFIG_HIGHMEM */ - - /* - * Fixed mappings: - */ - vaddr = __fix_to_virt(__end_of_fixed_addresses - 1); - pmd = pmd_off_k(vaddr); - fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!fixmap_pmd_p) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); - set_pmd(pmd, __pmd(__pa(fixmap_pmd_p) + _PAGE_KERNEL_TABLE)); - -#ifdef CONFIG_HIGHMEM - /* - * Permanent kmaps: - */ - vaddr = PKMAP_BASE; - - pmd = pmd_off_k(vaddr); - pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!pte) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); - set_pmd(pmd, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE)); - pkmap_page_table = pte; -#endif /* CONFIG_HIGHMEM */ -} - -/* - * paging_init() sets up the page tables, initialises the zone memory - * maps, and sets up the zero page, bad page and bad page tables. - */ -void __init paging_init(void) -{ - int i; - void *zero_page; - - pr_info("Setting up paging and PTEs.\n"); - /* clear out the init_mm.pgd that will contain the kernel's mappings */ - for (i = 0; i < PTRS_PER_PGD; i++) - swapper_pg_dir[i] = __pgd(1); - - map_ram(); - - fixedrange_init(); - - /* allocate space for empty_zero_page */ - zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!zero_page) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); - zone_sizes_init(); - - empty_zero_page = virt_to_page(zero_page); - flush_dcache_page(empty_zero_page); -} - -static inline void __init free_highmem(void) -{ -#ifdef CONFIG_HIGHMEM - unsigned long pfn; - for (pfn = PFN_UP(__pa(high_memory)); pfn < max_pfn; pfn++) { - phys_addr_t paddr = (phys_addr_t) pfn << PAGE_SHIFT; - if (!memblock_is_reserved(paddr)) - free_highmem_page(pfn_to_page(pfn)); - } -#endif -} - -static void __init set_max_mapnr_init(void) -{ - max_mapnr = max_pfn; -} - -/* - * mem_init() marks the free areas in the mem_map and tells us how much - * memory is free. This is done after various parts of the system have - * claimed their memory after the kernel image. - */ -void __init mem_init(void) -{ - phys_addr_t memory_start = memblock_start_of_DRAM(); - BUG_ON(!mem_map); - set_max_mapnr_init(); - - free_highmem(); - - /* this will put all low memory onto the freelists */ - memblock_free_all(); - - pr_info("virtual kernel memory layout:\n" - " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" -#ifdef CONFIG_HIGHMEM - " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" -#endif - " consist : 0x%08lx - 0x%08lx (%4ld MB)\n" - " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" - " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" - " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" - " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" - " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", - FIXADDR_START, FIXADDR_TOP, (FIXADDR_TOP - FIXADDR_START) >> 10, -#ifdef CONFIG_HIGHMEM - PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE, - (LAST_PKMAP * PAGE_SIZE) >> 10, -#endif - CONSISTENT_BASE, CONSISTENT_END, - ((CONSISTENT_END) - (CONSISTENT_BASE)) >> 20, VMALLOC_START, - (unsigned long)VMALLOC_END, (VMALLOC_END - VMALLOC_START) >> 20, - (unsigned long)__va(memory_start), (unsigned long)high_memory, - ((unsigned long)high_memory - - (unsigned long)__va(memory_start)) >> 20, - (unsigned long)&__init_begin, (unsigned long)&__init_end, - ((unsigned long)&__init_end - - (unsigned long)&__init_begin) >> 10, (unsigned long)&_etext, - (unsigned long)&_edata, - ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, - (unsigned long)&_text, (unsigned long)&_etext, - ((unsigned long)&_etext - (unsigned long)&_text) >> 10); - - /* - * Check boundaries twice: Some fundamental inconsistencies can - * be detected at build time already. - */ -#ifdef CONFIG_HIGHMEM - BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START); - BUILD_BUG_ON((CONSISTENT_END) > PKMAP_BASE); -#endif - BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE); - BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END); - -#ifdef CONFIG_HIGHMEM - BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START); - BUG_ON(CONSISTENT_END > PKMAP_BASE); -#endif - BUG_ON(VMALLOC_END > CONSISTENT_BASE); - BUG_ON(VMALLOC_START >= VMALLOC_END); - BUG_ON((unsigned long)high_memory > VMALLOC_START); - - return; -} - -void __set_fixmap(enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags) -{ - unsigned long addr = __fix_to_virt(idx); - pte_t *pte; - - BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); - - pte = (pte_t *)&fixmap_pmd_p[pte_index(addr)]; - - if (pgprot_val(flags)) { - set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); - } else { - pte_clear(&init_mm, addr, pte); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); - } -} diff --git a/arch/nds32/mm/mm-nds32.c b/arch/nds32/mm/mm-nds32.c deleted file mode 100644 index f2778f2b39f6..000000000000 --- a/arch/nds32/mm/mm-nds32.c +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/init_task.h> - -#define __HAVE_ARCH_PGD_FREE -#include <asm/pgalloc.h> - -#define FIRST_KERNEL_PGD_NR (USER_PTRS_PER_PGD) - -/* - * need to get a page for level 1 - */ - -pgd_t *pgd_alloc(struct mm_struct *mm) -{ - pgd_t *new_pgd, *init_pgd; - int i; - - new_pgd = (pgd_t *) __get_free_pages(GFP_KERNEL, 0); - if (!new_pgd) - return NULL; - for (i = 0; i < PTRS_PER_PGD; i++) { - (*new_pgd) = 1; - new_pgd++; - } - new_pgd -= PTRS_PER_PGD; - - init_pgd = pgd_offset_k(0); - - memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, - (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); - - cpu_dcache_wb_range((unsigned long)new_pgd, - (unsigned long)new_pgd + - PTRS_PER_PGD * sizeof(pgd_t)); - inc_lruvec_page_state(virt_to_page((unsigned long *)new_pgd), - NR_PAGETABLE); - - return new_pgd; -} - -void pgd_free(struct mm_struct *mm, pgd_t * pgd) -{ - pmd_t *pmd; - struct page *pte; - - if (!pgd) - return; - - pmd = (pmd_t *) pgd; - if (pmd_none(*pmd)) - goto free; - if (pmd_bad(*pmd)) { - pmd_ERROR(*pmd); - pmd_clear(pmd); - goto free; - } - - pte = pmd_page(*pmd); - pmd_clear(pmd); - dec_lruvec_page_state(virt_to_page((unsigned long *)pgd), NR_PAGETABLE); - pte_free(mm, pte); - mm_dec_nr_ptes(mm); - pmd_free(mm, pmd); -free: - free_pages((unsigned long)pgd, 0); -} - -/* - * In order to soft-boot, we need to insert a 1:1 mapping in place of - * the user-mode pages. This will then ensure that we have predictable - * results when turning the mmu off - */ -void setup_mm_for_reboot(char mode) -{ - unsigned long pmdval; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - int i; - - if (current->mm && current->mm->pgd) - pgd = current->mm->pgd; - else - pgd = init_mm.pgd; - - for (i = 0; i < USER_PTRS_PER_PGD; i++) { - pmdval = (i << PGDIR_SHIFT); - p4d = p4d_offset(pgd, i << PGDIR_SHIFT); - pud = pud_offset(p4d, i << PGDIR_SHIFT); - pmd = pmd_offset(pud + i, i << PGDIR_SHIFT); - set_pmd(pmd, __pmd(pmdval)); - } -} diff --git a/arch/nds32/mm/mmap.c b/arch/nds32/mm/mmap.c deleted file mode 100644 index 1bdf5e7d1b43..000000000000 --- a/arch/nds32/mm/mmap.c +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/sched.h> -#include <linux/mman.h> -#include <linux/shm.h> - -#define COLOUR_ALIGN(addr,pgoff) \ - ((((addr)+SHMLBA-1)&~(SHMLBA-1)) + \ - (((pgoff)<<PAGE_SHIFT) & (SHMLBA-1))) - -/* - * We need to ensure that shared mappings are correctly aligned to - * avoid aliasing issues with VIPT caches. We need to ensure that - * a specific page of an object is always mapped at a multiple of - * SHMLBA bytes. - * - * We unconditionally provide this function for all cases, however - * in the VIVT case, we optimise out the alignment rules. - */ -unsigned long -arch_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags) -{ - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - int do_align = 0; - struct vm_unmapped_area_info info; - int aliasing = 0; - if(IS_ENABLED(CONFIG_CPU_CACHE_ALIASING)) - aliasing = 1; - - /* - * We only need to do colour alignment if either the I or D - * caches alias. - */ - if (aliasing) - do_align = filp || (flags & MAP_SHARED); - - /* - * We enforce the MAP_FIXED case. - */ - if (flags & MAP_FIXED) { - if (aliasing && flags & MAP_SHARED && - (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) - return -EINVAL; - return addr; - } - - if (len > TASK_SIZE) - return -ENOMEM; - - if (addr) { - if (do_align) - addr = COLOUR_ALIGN(addr, pgoff); - else - addr = PAGE_ALIGN(addr); - - vma = find_vma(mm, addr); - if (TASK_SIZE - len >= addr && - (!vma || addr + len <= vm_start_gap(vma))) - return addr; - } - - info.flags = 0; - info.length = len; - info.low_limit = mm->mmap_base; - info.high_limit = TASK_SIZE; - info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; - return vm_unmapped_area(&info); -} diff --git a/arch/nds32/mm/proc.c b/arch/nds32/mm/proc.c deleted file mode 100644 index 848c845f5f33..000000000000 --- a/arch/nds32/mm/proc.c +++ /dev/null @@ -1,536 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <asm/nds32.h> -#include <asm/tlbflush.h> -#include <asm/cacheflush.h> -#include <asm/l2_cache.h> -#include <nds32_intrinsic.h> - -#include <asm/cache_info.h> -extern struct cache_info L1_cache_info[2]; - -int va_kernel_present(unsigned long addr) -{ - pmd_t *pmd; - pte_t *ptep, pte; - - pmd = pmd_off_k(addr); - if (!pmd_none(*pmd)) { - ptep = pte_offset_map(pmd, addr); - pte = *ptep; - if (pte_present(pte)) - return pte; - } - return 0; -} - -pte_t va_present(struct mm_struct * mm, unsigned long addr) -{ - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *ptep, pte; - - pgd = pgd_offset(mm, addr); - if (!pgd_none(*pgd)) { - p4d = p4d_offset(pgd, addr); - if (!p4d_none(*p4d)) { - pud = pud_offset(p4d, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) { - ptep = pte_offset_map(pmd, addr); - pte = *ptep; - if (pte_present(pte)) - return pte; - } - } - } - } - return 0; - -} - -int va_readable(struct pt_regs *regs, unsigned long addr) -{ - struct mm_struct *mm = current->mm; - pte_t pte; - int ret = 0; - - if (user_mode(regs)) { - /* user mode */ - pte = va_present(mm, addr); - if (!pte && pte_read(pte)) - ret = 1; - } else { - /* superuser mode is always readable, so we can only - * check it is present or not*/ - return (! !va_kernel_present(addr)); - } - return ret; -} - -int va_writable(struct pt_regs *regs, unsigned long addr) -{ - struct mm_struct *mm = current->mm; - pte_t pte; - int ret = 0; - - if (user_mode(regs)) { - /* user mode */ - pte = va_present(mm, addr); - if (!pte && pte_write(pte)) - ret = 1; - } else { - /* superuser mode */ - pte = va_kernel_present(addr); - if (!pte && pte_kernel_write(pte)) - ret = 1; - } - return ret; -} - -/* - * All - */ -void cpu_icache_inval_all(void) -{ - unsigned long end, line_size; - - line_size = L1_cache_info[ICACHE].line_size; - end = - line_size * L1_cache_info[ICACHE].ways * L1_cache_info[ICACHE].sets; - - do { - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end)); - } while (end > 0); - __nds32__isb(); -} - -void cpu_dcache_inval_all(void) -{ - __nds32__cctl_l1d_invalall(); -} - -#ifdef CONFIG_CACHE_L2 -void dcache_wb_all_level(void) -{ - unsigned long flags, cmd; - local_irq_save(flags); - __nds32__cctl_l1d_wball_alvl(); - /* Section 1: Ensure the section 2 & 3 program code execution after */ - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); - - /* Section 2: Confirm the writeback all level is done in CPU and L2C */ - cmd = CCTL_CMD_L2_SYNC; - L2_CMD_RDY(); - L2C_W_REG(L2_CCTL_CMD_OFF, cmd); - L2_CMD_RDY(); - - /* Section 3: Writeback whole L2 cache */ - cmd = CCTL_ALL_CMD | CCTL_CMD_L2_IX_WB; - L2_CMD_RDY(); - L2C_W_REG(L2_CCTL_CMD_OFF, cmd); - L2_CMD_RDY(); - __nds32__msync_all(); - local_irq_restore(flags); -} -EXPORT_SYMBOL(dcache_wb_all_level); -#endif - -void cpu_dcache_wb_all(void) -{ - __nds32__cctl_l1d_wball_one_lvl(); - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); -} - -void cpu_dcache_wbinval_all(void) -{ -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - unsigned long flags; - local_irq_save(flags); -#endif - cpu_dcache_wb_all(); - cpu_dcache_inval_all(); -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - local_irq_restore(flags); -#endif -} - -/* - * Page - */ -void cpu_icache_inval_page(unsigned long start) -{ - unsigned long line_size, end; - - line_size = L1_cache_info[ICACHE].line_size; - end = start + PAGE_SIZE; - - do { - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end)); - } while (end != start); - __nds32__isb(); -} - -void cpu_dcache_inval_page(unsigned long start) -{ - unsigned long line_size, end; - - line_size = L1_cache_info[DCACHE].line_size; - end = start + PAGE_SIZE; - - do { - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - } while (end != start); -} - -void cpu_dcache_wb_page(unsigned long start) -{ -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - unsigned long line_size, end; - - line_size = L1_cache_info[DCACHE].line_size; - end = start + PAGE_SIZE; - - do { - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); - end -= line_size; - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); - } while (end != start); - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); -#endif -} - -void cpu_dcache_wbinval_page(unsigned long start) -{ - unsigned long line_size, end; - - line_size = L1_cache_info[DCACHE].line_size; - end = start + PAGE_SIZE; - - do { - end -= line_size; -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); -#endif - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); -#endif - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); -#endif - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - end -= line_size; -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end)); -#endif - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end)); - } while (end != start); - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); -} - -void cpu_cache_wbinval_page(unsigned long page, int flushi) -{ - cpu_dcache_wbinval_page(page); - if (flushi) - cpu_icache_inval_page(page); -} - -/* - * Range - */ -void cpu_icache_inval_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - - line_size = L1_cache_info[ICACHE].line_size; - - while (end > start) { - __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (start)); - start += line_size; - } - __nds32__isb(); -} - -void cpu_dcache_inval_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - - line_size = L1_cache_info[DCACHE].line_size; - - while (end > start) { - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start)); - start += line_size; - } -} - -void cpu_dcache_wb_range(unsigned long start, unsigned long end) -{ -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - unsigned long line_size; - - line_size = L1_cache_info[DCACHE].line_size; - - while (end > start) { - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start)); - start += line_size; - } - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); -#endif -} - -void cpu_dcache_wbinval_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - - line_size = L1_cache_info[DCACHE].line_size; - - while (end > start) { -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start)); -#endif - __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start)); - start += line_size; - } - __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0); -} - -void cpu_cache_wbinval_range(unsigned long start, unsigned long end, int flushi) -{ - unsigned long line_size, align_start, align_end; - - line_size = L1_cache_info[DCACHE].line_size; - align_start = start & ~(line_size - 1); - align_end = (end + line_size - 1) & ~(line_size - 1); - cpu_dcache_wbinval_range(align_start, align_end); - - if (flushi) { - line_size = L1_cache_info[ICACHE].line_size; - align_start = start & ~(line_size - 1); - align_end = (end + line_size - 1) & ~(line_size - 1); - cpu_icache_inval_range(align_start, align_end); - } -} - -void cpu_cache_wbinval_range_check(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - bool flushi, bool wbd) -{ - unsigned long line_size, t_start, t_end; - - if (!flushi && !wbd) - return; - line_size = L1_cache_info[DCACHE].line_size; - start = start & ~(line_size - 1); - end = (end + line_size - 1) & ~(line_size - 1); - - if ((end - start) > (8 * PAGE_SIZE)) { - if (wbd) - cpu_dcache_wbinval_all(); - if (flushi) - cpu_icache_inval_all(); - return; - } - - t_start = (start + PAGE_SIZE) & PAGE_MASK; - t_end = ((end - 1) & PAGE_MASK); - - if ((start & PAGE_MASK) == t_end) { - if (va_present(vma->vm_mm, start)) { - if (wbd) - cpu_dcache_wbinval_range(start, end); - if (flushi) - cpu_icache_inval_range(start, end); - } - return; - } - - if (va_present(vma->vm_mm, start)) { - if (wbd) - cpu_dcache_wbinval_range(start, t_start); - if (flushi) - cpu_icache_inval_range(start, t_start); - } - - if (va_present(vma->vm_mm, end - 1)) { - if (wbd) - cpu_dcache_wbinval_range(t_end, end); - if (flushi) - cpu_icache_inval_range(t_end, end); - } - - while (t_start < t_end) { - if (va_present(vma->vm_mm, t_start)) { - if (wbd) - cpu_dcache_wbinval_page(t_start); - if (flushi) - cpu_icache_inval_page(t_start); - } - t_start += PAGE_SIZE; - } -} - -#ifdef CONFIG_CACHE_L2 -static inline void cpu_l2cache_op(unsigned long start, unsigned long end, unsigned long op) -{ - if (atl2c_base) { - unsigned long p_start = __pa(start); - unsigned long p_end = __pa(end); - unsigned long cmd; - unsigned long line_size; - /* TODO Can Use PAGE Mode to optimize if range large than PAGE_SIZE */ - line_size = L2_CACHE_LINE_SIZE(); - p_start = p_start & (~(line_size - 1)); - p_end = (p_end + line_size - 1) & (~(line_size - 1)); - cmd = - (p_start & ~(line_size - 1)) | op | - CCTL_SINGLE_CMD; - do { - L2_CMD_RDY(); - L2C_W_REG(L2_CCTL_CMD_OFF, cmd); - cmd += line_size; - p_start += line_size; - } while (p_end > p_start); - cmd = CCTL_CMD_L2_SYNC; - L2_CMD_RDY(); - L2C_W_REG(L2_CCTL_CMD_OFF, cmd); - L2_CMD_RDY(); - } -} -#else -#define cpu_l2cache_op(start,end,op) do { } while (0) -#endif -/* - * DMA - */ -void cpu_dma_wb_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - unsigned long flags; - line_size = L1_cache_info[DCACHE].line_size; - start = start & (~(line_size - 1)); - end = (end + line_size - 1) & (~(line_size - 1)); - if (unlikely(start == end)) - return; - - local_irq_save(flags); - cpu_dcache_wb_range(start, end); - cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WB); - __nds32__msync_all(); - local_irq_restore(flags); -} - -void cpu_dma_inval_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - unsigned long old_start = start; - unsigned long old_end = end; - unsigned long flags; - line_size = L1_cache_info[DCACHE].line_size; - start = start & (~(line_size - 1)); - end = (end + line_size - 1) & (~(line_size - 1)); - if (unlikely(start == end)) - return; - local_irq_save(flags); - if (start != old_start) { - cpu_dcache_wbinval_range(start, start + line_size); - cpu_l2cache_op(start, start + line_size, CCTL_CMD_L2_PA_WBINVAL); - } - if (end != old_end) { - cpu_dcache_wbinval_range(end - line_size, end); - cpu_l2cache_op(end - line_size, end, CCTL_CMD_L2_PA_WBINVAL); - } - cpu_dcache_inval_range(start, end); - cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_INVAL); - __nds32__msync_all(); - local_irq_restore(flags); - -} - -void cpu_dma_wbinval_range(unsigned long start, unsigned long end) -{ - unsigned long line_size; - unsigned long flags; - line_size = L1_cache_info[DCACHE].line_size; - start = start & (~(line_size - 1)); - end = (end + line_size - 1) & (~(line_size - 1)); - if (unlikely(start == end)) - return; - - local_irq_save(flags); - cpu_dcache_wbinval_range(start, end); - cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WBINVAL); - __nds32__msync_all(); - local_irq_restore(flags); -} - -void cpu_proc_init(void) -{ -} - -void cpu_proc_fin(void) -{ -} - -void cpu_do_idle(void) -{ - __nds32__standby_no_wake_grant(); -} - -void cpu_reset(unsigned long reset) -{ - u32 tmp; - GIE_DISABLE(); - tmp = __nds32__mfsr(NDS32_SR_CACHE_CTL); - tmp &= ~(CACHE_CTL_mskIC_EN | CACHE_CTL_mskDC_EN); - __nds32__mtsr_isb(tmp, NDS32_SR_CACHE_CTL); - cpu_dcache_wbinval_all(); - cpu_icache_inval_all(); - - __asm__ __volatile__("jr.toff %0\n\t"::"r"(reset)); -} - -void cpu_switch_mm(struct mm_struct *mm) -{ - unsigned long cid; - cid = __nds32__mfsr(NDS32_SR_TLB_MISC); - cid = (cid & ~TLB_MISC_mskCID) | mm->context.id; - __nds32__mtsr_dsb(cid, NDS32_SR_TLB_MISC); - __nds32__mtsr_isb(__pa(mm->pgd), NDS32_SR_L1_PPTB); -} diff --git a/arch/nds32/mm/tlb.c b/arch/nds32/mm/tlb.c deleted file mode 100644 index dd41f5e0712f..000000000000 --- a/arch/nds32/mm/tlb.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2005-2017 Andes Technology Corporation - -#include <linux/spinlock_types.h> -#include <linux/mm.h> -#include <linux/sched.h> -#include <asm/nds32.h> -#include <nds32_intrinsic.h> - -unsigned int cpu_last_cid = { TLB_MISC_mskCID + (2 << TLB_MISC_offCID) }; - -DEFINE_SPINLOCK(cid_lock); - -void local_flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - unsigned long flags, ocid, ncid; - - if ((end - start) > 0x400000) { - __nds32__tlbop_flua(); - __nds32__isb(); - return; - } - - spin_lock_irqsave(&cid_lock, flags); - ocid = __nds32__mfsr(NDS32_SR_TLB_MISC); - ncid = (ocid & ~TLB_MISC_mskCID) | vma->vm_mm->context.id; - __nds32__mtsr_dsb(ncid, NDS32_SR_TLB_MISC); - while (start < end) { - __nds32__tlbop_inv(start); - __nds32__isb(); - start += PAGE_SIZE; - } - __nds32__mtsr_dsb(ocid, NDS32_SR_TLB_MISC); - spin_unlock_irqrestore(&cid_lock, flags); -} - -void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) -{ - unsigned long flags, ocid, ncid; - - spin_lock_irqsave(&cid_lock, flags); - ocid = __nds32__mfsr(NDS32_SR_TLB_MISC); - ncid = (ocid & ~TLB_MISC_mskCID) | vma->vm_mm->context.id; - __nds32__mtsr_dsb(ncid, NDS32_SR_TLB_MISC); - __nds32__tlbop_inv(addr); - __nds32__isb(); - __nds32__mtsr_dsb(ocid, NDS32_SR_TLB_MISC); - spin_unlock_irqrestore(&cid_lock, flags); -} diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 33fd06f5fa41..4167f1eb4cd8 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -24,7 +24,6 @@ config NIOS2 select USB_ARCH_HAS_HCD if USB_SUPPORT select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE if MMU - select SET_FS config GENERIC_CSUM def_bool y diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h index 4a995fa628ee..262d0609268c 100644 --- a/arch/nios2/include/asm/pgtable.h +++ b/arch/nios2/include/asm/pgtable.h @@ -235,6 +235,7 @@ static inline void pte_clear(struct mm_struct *mm, * and a page entry and page directory to the page they refer to. */ #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) +#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) static inline unsigned long pmd_page_vaddr(pmd_t pmd) diff --git a/arch/nios2/include/asm/thread_info.h b/arch/nios2/include/asm/thread_info.h index 272d2c72a727..bcc0e9915ebd 100644 --- a/arch/nios2/include/asm/thread_info.h +++ b/arch/nios2/include/asm/thread_info.h @@ -26,10 +26,6 @@ #ifndef __ASSEMBLY__ -typedef struct { - unsigned long seg; -} mm_segment_t; - /* * low level task data that entry.S needs immediate access to * - this struct should fit entirely inside of one cache line @@ -42,10 +38,6 @@ struct thread_info { unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ int preempt_count; /* 0 => preemptable,<0 => BUG */ - mm_segment_t addr_limit; /* thread address space: - 0-0x7FFFFFFF for user-thead - 0-0xFFFFFFFF for kernel-thread - */ struct pt_regs *regs; }; @@ -60,7 +52,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } /* how to get the thread information struct from C */ diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index ba9340e96fd4..b8299082adbe 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -18,31 +18,10 @@ #include <asm/page.h> #include <asm/extable.h> - -/* - * Segment stuff - */ -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) -#define USER_DS MAKE_MM_SEG(0x80000000UL) -#define KERNEL_DS MAKE_MM_SEG(0) - - -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(seg) (current_thread_info()->addr_limit = (seg)) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -#define __access_ok(addr, len) \ - (((signed long)(((long)get_fs().seg) & \ - ((long)(addr) | (((long)(addr)) + (len)) | (len)))) == 0) - -#define access_ok(addr, len) \ - likely(__access_ok((unsigned long)(addr), (unsigned long)(len))) +#include <asm-generic/access_ok.h> # define __EX_TABLE_SECTION ".section __ex_table,\"a\"\n" -#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE) - /* * Zero Userspace */ @@ -88,6 +67,7 @@ extern __must_check long strnlen_user(const char __user *s, long n); /* Optimized macros */ #define __get_user_asm(val, insn, addr, err) \ { \ + unsigned long __gu_val; \ __asm__ __volatile__( \ " movi %0, %3\n" \ "1: " insn " %1, 0(%2)\n" \ @@ -96,14 +76,20 @@ extern __must_check long strnlen_user(const char __user *s, long n); " .section __ex_table,\"a\"\n" \ " .word 1b, 2b\n" \ " .previous" \ - : "=&r" (err), "=r" (val) \ + : "=&r" (err), "=r" (__gu_val) \ : "r" (addr), "i" (-EFAULT)); \ + val = (__force __typeof__(*(addr)))__gu_val; \ } -#define __get_user_unknown(val, size, ptr, err) do { \ +extern void __get_user_unknown(void); + +#define __get_user_8(val, ptr, err) do { \ + u64 __val = 0; \ err = 0; \ - if (__copy_from_user(&(val), ptr, size)) { \ + if (raw_copy_from_user(&(__val), ptr, sizeof(val))) { \ err = -EFAULT; \ + } else { \ + val = (typeof(val))(typeof((val) - (val)))__val; \ } \ } while (0) @@ -119,8 +105,11 @@ do { \ case 4: \ __get_user_asm(val, "ldw", ptr, err); \ break; \ + case 8: \ + __get_user_8(val, ptr, err); \ + break; \ default: \ - __get_user_unknown(val, size, ptr, err); \ + __get_user_unknown(); \ break; \ } \ } while (0) @@ -129,9 +118,7 @@ do { \ ({ \ long __gu_err = -EFAULT; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ - unsigned long __gu_val = 0; \ - __get_user_common(__gu_val, sizeof(*(ptr)), __gu_ptr, __gu_err);\ - (x) = (__force __typeof__(x))__gu_val; \ + __get_user_common(x, sizeof(*(ptr)), __gu_ptr, __gu_err); \ __gu_err; \ }) @@ -139,11 +126,9 @@ do { \ ({ \ long __gu_err = -EFAULT; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ - unsigned long __gu_val = 0; \ if (access_ok( __gu_ptr, sizeof(*__gu_ptr))) \ - __get_user_common(__gu_val, sizeof(*__gu_ptr), \ + __get_user_common(x, sizeof(*__gu_ptr), \ __gu_ptr, __gu_err); \ - (x) = (__force __typeof__(x))__gu_val; \ __gu_err; \ }) @@ -161,34 +146,44 @@ do { \ : "r" (val), "r" (ptr), "i" (-EFAULT)); \ } -#define put_user(x, ptr) \ +#define __put_user_common(__pu_val, __pu_ptr) \ ({ \ long __pu_err = -EFAULT; \ - __typeof__(*(ptr)) __user *__pu_ptr = (ptr); \ - __typeof__(*(ptr)) __pu_val = (__typeof(*ptr))(x); \ - if (access_ok(__pu_ptr, sizeof(*__pu_ptr))) { \ - switch (sizeof(*__pu_ptr)) { \ - case 1: \ - __put_user_asm(__pu_val, "stb", __pu_ptr, __pu_err); \ - break; \ - case 2: \ - __put_user_asm(__pu_val, "sth", __pu_ptr, __pu_err); \ - break; \ - case 4: \ - __put_user_asm(__pu_val, "stw", __pu_ptr, __pu_err); \ - break; \ - default: \ - /* XXX: This looks wrong... */ \ - __pu_err = 0; \ - if (copy_to_user(__pu_ptr, &(__pu_val), \ - sizeof(*__pu_ptr))) \ - __pu_err = -EFAULT; \ - break; \ - } \ + switch (sizeof(*__pu_ptr)) { \ + case 1: \ + __put_user_asm(__pu_val, "stb", __pu_ptr, __pu_err); \ + break; \ + case 2: \ + __put_user_asm(__pu_val, "sth", __pu_ptr, __pu_err); \ + break; \ + case 4: \ + __put_user_asm(__pu_val, "stw", __pu_ptr, __pu_err); \ + break; \ + default: \ + /* XXX: This looks wrong... */ \ + __pu_err = 0; \ + if (__copy_to_user(__pu_ptr, &(__pu_val), \ + sizeof(*__pu_ptr))) \ + __pu_err = -EFAULT; \ + break; \ } \ __pu_err; \ }) -#define __put_user(x, ptr) put_user(x, ptr) +#define __put_user(x, ptr) \ +({ \ + __auto_type __pu_ptr = (ptr); \ + typeof(*__pu_ptr) __pu_val = (typeof(*__pu_ptr))(x); \ + __put_user_common(__pu_val, __pu_ptr); \ +}) + +#define put_user(x, ptr) \ +({ \ + __auto_type __pu_ptr = (ptr); \ + typeof(*__pu_ptr) __pu_val = (typeof(*__pu_ptr))(x); \ + access_ok(__pu_ptr, sizeof(*__pu_ptr)) ? \ + __put_user_common(__pu_val, __pu_ptr) : \ + -EFAULT; \ +}) #endif /* _ASM_NIOS2_UACCESS_H */ diff --git a/arch/nios2/kernel/signal.c b/arch/nios2/kernel/signal.c index 2009ae2d3c3b..386e46443b60 100644 --- a/arch/nios2/kernel/signal.c +++ b/arch/nios2/kernel/signal.c @@ -36,10 +36,10 @@ struct rt_sigframe { static inline int rt_restore_ucontext(struct pt_regs *regs, struct switch_stack *sw, - struct ucontext *uc, int *pr2) + struct ucontext __user *uc, int *pr2) { int temp; - unsigned long *gregs = uc->uc_mcontext.gregs; + unsigned long __user *gregs = uc->uc_mcontext.gregs; int err; /* Always make any pending restarted system calls return -EINTR */ @@ -102,10 +102,11 @@ asmlinkage int do_rt_sigreturn(struct switch_stack *sw) { struct pt_regs *regs = (struct pt_regs *)(sw + 1); /* Verify, can we follow the stack back */ - struct rt_sigframe *frame = (struct rt_sigframe *) regs->sp; + struct rt_sigframe __user *frame; sigset_t set; int rval; + frame = (struct rt_sigframe __user *) regs->sp; if (!access_ok(frame, sizeof(*frame))) goto badframe; @@ -124,10 +125,10 @@ badframe: return 0; } -static inline int rt_setup_ucontext(struct ucontext *uc, struct pt_regs *regs) +static inline int rt_setup_ucontext(struct ucontext __user *uc, struct pt_regs *regs) { struct switch_stack *sw = (struct switch_stack *)regs - 1; - unsigned long *gregs = uc->uc_mcontext.gregs; + unsigned long __user *gregs = uc->uc_mcontext.gregs; int err = 0; err |= __put_user(MCONTEXT_VERSION, &uc->uc_mcontext.version); @@ -162,8 +163,9 @@ static inline int rt_setup_ucontext(struct ucontext *uc, struct pt_regs *regs) return err; } -static inline void *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, - size_t frame_size) +static inline void __user *get_sigframe(struct ksignal *ksig, + struct pt_regs *regs, + size_t frame_size) { unsigned long usp; @@ -174,13 +176,13 @@ static inline void *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, usp = sigsp(usp, ksig); /* Verify, is it 32 or 64 bit aligned */ - return (void *)((usp - frame_size) & -8UL); + return (void __user *)((usp - frame_size) & -8UL); } static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { - struct rt_sigframe *frame; + struct rt_sigframe __user *frame; int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index f724b3f1aeed..0d68adf6e02b 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -36,7 +36,6 @@ config OPENRISC select ARCH_WANT_FRAME_POINTERS select GENERIC_IRQ_MULTI_HANDLER select MMU_GATHER_NO_RANGE if MMU - select SET_FS select TRACE_IRQFLAGS_SUPPORT config CPU_BIG_ENDIAN diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h index cdd657f80bfa..c3abbf71e09f 100644 --- a/arch/openrisc/include/asm/pgtable.h +++ b/arch/openrisc/include/asm/pgtable.h @@ -361,6 +361,7 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) pmd_val(*pmdp) = _KERNPG_TABLE | (unsigned long) ptep; } +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) static inline unsigned long pmd_page_vaddr(pmd_t pmd) diff --git a/arch/openrisc/include/asm/thread_info.h b/arch/openrisc/include/asm/thread_info.h index 659834ab87fa..4af3049c34c2 100644 --- a/arch/openrisc/include/asm/thread_info.h +++ b/arch/openrisc/include/asm/thread_info.h @@ -40,18 +40,12 @@ */ #ifndef __ASSEMBLY__ -typedef unsigned long mm_segment_t; - struct thread_info { struct task_struct *task; /* main task structure */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ __s32 preempt_count; /* 0 => preemptable, <0 => BUG */ - mm_segment_t addr_limit; /* thread address space: - 0-0x7FFFFFFF for user-thead - 0-0xFFFFFFFF for kernel-thread - */ __u8 supervisor_stack[0]; /* saved context data */ @@ -71,7 +65,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ .ksp = 0, \ } diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 120f5005461b..d6500a374e18 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -22,44 +22,7 @@ #include <linux/string.h> #include <asm/page.h> #include <asm/extable.h> - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -/* addr_limit is the maximum accessible address for the task. we misuse - * the KERNEL_DS and USER_DS values to both assign and compare the - * addr_limit values through the equally misnamed get/set_fs macros. - * (see above) - */ - -#define KERNEL_DS (~0UL) - -#define USER_DS (TASK_SIZE) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define uaccess_kernel() (get_fs() == KERNEL_DS) - -/* Ensure that the range from addr to addr+size is all within the process' - * address space - */ -static inline int __range_ok(unsigned long addr, unsigned long size) -{ - const mm_segment_t fs = get_fs(); - - return size <= fs && addr <= (fs - size); -} - -#define access_ok(addr, size) \ -({ \ - __chk_user_ptr(addr); \ - __range_ok((unsigned long)(addr), (size)); \ -}) +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They automatically @@ -268,9 +231,6 @@ clear_user(void __user *addr, unsigned long size) return size; } -#define user_addr_max() \ - (uaccess_kernel() ? ~0UL : TASK_SIZE) - extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strnlen_user(const char __user *str, long n); diff --git a/arch/openrisc/mm/init.c b/arch/openrisc/mm/init.c index 97305bde1b16..3a021ab6f1ae 100644 --- a/arch/openrisc/mm/init.c +++ b/arch/openrisc/mm/init.c @@ -38,8 +38,6 @@ int mem_init_done; -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); - static void __init zone_sizes_init(void) { unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 43c1c880def6..00cb889bd9a6 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 config PARISC def_bool y + select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_OFF_T if !64BIT select ARCH_MIGHT_HAVE_PC_PARPORT select HAVE_FUNCTION_TRACER @@ -10,10 +11,12 @@ config PARISC select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL + select ARCH_HAS_PTE_SPECIAL select ARCH_NO_SG_CHAIN select ARCH_SUPPORTS_HUGETLBFS if PA20 select ARCH_SUPPORTS_MEMORY_FAILURE select ARCH_STACKWALK + select ARCH_HAS_DEBUG_VM_PGTABLE select HAVE_RELIABLE_STACKTRACE select DMA_OPS select RTC_CLASS @@ -259,18 +262,6 @@ config PARISC_PAGE_SIZE_64KB endchoice -config PARISC_SELF_EXTRACT - bool "Build kernel as self-extracting executable" - default y - help - Say Y if you want to build the parisc kernel as a kind of - self-extracting executable. - - If you say N here, the kernel will be compressed with gzip - which can be loaded by the palo bootloader directly too. - - If you don't know what to do here, say Y. - config SMP bool "Symmetric multi-processing support" help diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index 82d77f4b0d08..2a9387a93592 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -15,12 +15,8 @@ # Mike Shaver, Helge Deller and Martin K. Petersen # -ifdef CONFIG_PARISC_SELF_EXTRACT boot := arch/parisc/boot KBUILD_IMAGE := $(boot)/bzImage -else -KBUILD_IMAGE := vmlinuz -endif NM = sh $(srctree)/arch/parisc/nm CHECKFLAGS += -D__hppa__=1 @@ -44,6 +40,16 @@ endif export LD_BFD +# Set default 32 bits cross compilers for vdso +CC_ARCHES_32 = hppa hppa2.0 hppa1.1 +CC_SUFFIXES = linux linux-gnu unknown-linux-gnu +CROSS32_COMPILE := $(call cc-cross-prefix, \ + $(foreach a,$(CC_ARCHES_32), \ + $(foreach s,$(CC_SUFFIXES),$(a)-$(s)-))) +CROSS32CC := $(CROSS32_COMPILE)gcc +export CROSS32CC + +# Set default cross compiler for kernel build ifdef cross_compiling ifeq ($(CROSS_COMPILE),) CC_SUFFIXES = linux linux-gnu unknown-linux-gnu @@ -155,14 +161,29 @@ Image: vmlinux bzImage: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ -ifdef CONFIG_PARISC_SELF_EXTRACT vmlinuz: bzImage $(OBJCOPY) $(boot)/bzImage $@ -else -vmlinuz: vmlinux - @$(KGZIP) -cf -9 $< > $@ + +ifeq ($(KBUILD_EXTMOD),) +# We need to generate vdso-offsets.h before compiling certain files in kernel/. +# In order to do that, we should use the archprepare target, but we can't since +# asm-offsets.h is included in some files used to generate vdso-offsets.h, and +# asm-offsets.h is built in prepare0, for which archprepare is a dependency. +# Therefore we need to generate the header after prepare0 has been made, hence +# this hack. +prepare: vdso_prepare +vdso_prepare: prepare0 + $(if $(CONFIG_64BIT),$(Q)$(MAKE) \ + $(build)=arch/parisc/kernel/vdso64 include/generated/vdso64-offsets.h) + $(Q)$(MAKE) $(build)=arch/parisc/kernel/vdso32 include/generated/vdso32-offsets.h endif +PHONY += vdso_install + +vdso_install: + $(Q)$(MAKE) $(build)=arch/parisc/kernel/vdso $@ + $(if $(CONFIG_COMPAT_VDSO), \ + $(Q)$(MAKE) $(build)=arch/parisc/kernel/vdso32 $@) install: $(CONFIG_SHELL) $(srctree)/arch/parisc/install.sh \ $(KERNELRELEASE) vmlinux System.map "$(INSTALL_PATH)" diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 53061cb2cf7f..a5fee10d76ee 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -210,7 +210,6 @@ CONFIG_TMPFS_XATTR=y CONFIG_NFS_FS=m # CONFIG_NFS_V2 is not set CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y diff --git a/arch/parisc/include/asm/agp.h b/arch/parisc/include/asm/agp.h index cb04470e63d0..14ae54cfd368 100644 --- a/arch/parisc/include/asm/agp.h +++ b/arch/parisc/include/asm/agp.h @@ -8,8 +8,8 @@ * */ -#define map_page_into_agp(page) /* nothing */ -#define unmap_page_from_agp(page) /* nothing */ +#define map_page_into_agp(page) do { } while (0) +#define unmap_page_from_agp(page) do { } while (0) #define flush_agp_cache() mb() /* GATT allocation. Returns/accepts GATT kernel virtual address. */ diff --git a/arch/parisc/include/asm/assembly.h b/arch/parisc/include/asm/assembly.h index 6369082c6c74..ea0cb318b13d 100644 --- a/arch/parisc/include/asm/assembly.h +++ b/arch/parisc/include/asm/assembly.h @@ -47,6 +47,12 @@ #define PRIV_USER 3 #define PRIV_KERNEL 0 +/* Space register used inside kernel */ +#define SR_KERNEL 0 +#define SR_TEMP1 1 +#define SR_TEMP2 2 +#define SR_USER 3 + #ifdef __ASSEMBLY__ #ifdef CONFIG_64BIT diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 0ec9cfc5131f..56ffd260c669 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -12,6 +12,14 @@ #include <asm/barrier.h> #include <linux/atomic.h> +/* compiler build environment sanity checks: */ +#if !defined(CONFIG_64BIT) && defined(__LP64__) +#error "Please use 'ARCH=parisc' to build the 32-bit kernel." +#endif +#if defined(CONFIG_64BIT) && !defined(__LP64__) +#error "Please use 'ARCH=parisc64' to build the 64-bit kernel." +#endif + /* See http://marc.theaimsgroup.com/?t=108826637900003 for discussion * on use of volatile and __*_bit() (set/clear/change): * *_bit() want use of volatile. diff --git a/arch/parisc/include/asm/cache.h b/arch/parisc/include/asm/cache.h index d53e9e27dba0..5032e758594e 100644 --- a/arch/parisc/include/asm/cache.h +++ b/arch/parisc/include/asm/cache.h @@ -39,16 +39,13 @@ extern int icache_stride; extern struct pdc_cache_info cache_info; void parisc_setup_cache_timing(void); -#define pdtlb(addr) asm volatile("pdtlb 0(%%sr1,%0)" \ +#define pdtlb(sr, addr) asm volatile("pdtlb 0(%%sr%0,%1)" \ ALTERNATIVE(ALT_COND_NO_SMP, INSN_PxTLB) \ - : : "r" (addr) : "memory") -#define pitlb(addr) asm volatile("pitlb 0(%%sr1,%0)" \ + : : "i"(sr), "r" (addr) : "memory") +#define pitlb(sr, addr) asm volatile("pitlb 0(%%sr%0,%1)" \ ALTERNATIVE(ALT_COND_NO_SMP, INSN_PxTLB) \ ALTERNATIVE(ALT_COND_NO_SPLIT_TLB, INSN_NOP) \ - : : "r" (addr) : "memory") -#define pdtlb_kernel(addr) asm volatile("pdtlb 0(%0)" \ - ALTERNATIVE(ALT_COND_NO_SMP, INSN_PxTLB) \ - : : "r" (addr) : "memory") + : : "i"(sr), "r" (addr) : "memory") #define asm_io_fdc(addr) asm volatile("fdc %%r0(%0)" \ ALTERNATIVE(ALT_COND_NO_DCACHE, INSN_NOP) \ diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index 859b8a34adcf..e8b4a03343d3 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -9,16 +9,11 @@ /* The usual comment is "Caches aren't brain-dead on the <architecture>". * Unfortunately, that doesn't apply to PA-RISC. */ -/* Internal implementation */ -void flush_data_cache_local(void *); /* flushes local data-cache only */ -void flush_instruction_cache_local(void *); /* flushes local code-cache only */ -#ifdef CONFIG_SMP -void flush_data_cache(void); /* flushes data-cache only (all processors) */ -void flush_instruction_cache(void); /* flushes i-cache only (all processors) */ -#else -#define flush_data_cache() flush_data_cache_local(NULL) -#define flush_instruction_cache() flush_instruction_cache_local(NULL) -#endif +#include <linux/jump_label.h> + +DECLARE_STATIC_KEY_TRUE(parisc_has_cache); +DECLARE_STATIC_KEY_TRUE(parisc_has_dcache); +DECLARE_STATIC_KEY_TRUE(parisc_has_icache); #define flush_cache_dup_mm(mm) flush_cache_mm(mm) diff --git a/arch/parisc/include/asm/current.h b/arch/parisc/include/asm/current.h index 568b739e42af..dc7aea07c3f3 100644 --- a/arch/parisc/include/asm/current.h +++ b/arch/parisc/include/asm/current.h @@ -2,14 +2,16 @@ #ifndef _ASM_PARISC_CURRENT_H #define _ASM_PARISC_CURRENT_H -#include <asm/special_insns.h> - #ifndef __ASSEMBLY__ struct task_struct; static __always_inline struct task_struct *get_current(void) { - return (struct task_struct *) mfctl(30); + struct task_struct *ts; + + /* do not use mfctl() macro as it is marked volatile */ + asm( "mfctl %%cr30,%0" : "=r" (ts) ); + return ts; } #define current get_current() diff --git a/arch/parisc/include/asm/elf.h b/arch/parisc/include/asm/elf.h index 3bd465a27791..cc426d365892 100644 --- a/arch/parisc/include/asm/elf.h +++ b/arch/parisc/include/asm/elf.h @@ -359,4 +359,19 @@ struct mm_struct; extern unsigned long arch_randomize_brk(struct mm_struct *); #define arch_randomize_brk arch_randomize_brk + +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +struct linux_binprm; +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int executable_stack); +#define VDSO_AUX_ENT(a, b) NEW_AUX_ENT(a, b) +#define VDSO_CURRENT_BASE current->mm->context.vdso_base + +#define ARCH_DLINFO \ +do { \ + if (VDSO_CURRENT_BASE) { \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, VDSO_CURRENT_BASE);\ + } \ +} while (0) + #endif diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h index b5835325d44b..3222206cb3ea 100644 --- a/arch/parisc/include/asm/futex.h +++ b/arch/parisc/include/asm/futex.h @@ -96,12 +96,6 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 val; unsigned long flags; - /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is - * our gateway page, and causes no end of trouble... - */ - if (uaccess_kernel() && !uaddr) - return -EFAULT; - if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; diff --git a/arch/parisc/include/asm/kprobes.h b/arch/parisc/include/asm/kprobes.h index 904034da4974..0a175ac87698 100644 --- a/arch/parisc/include/asm/kprobes.h +++ b/arch/parisc/include/asm/kprobes.h @@ -18,8 +18,9 @@ #include <linux/notifier.h> #define PARISC_KPROBES_BREAK_INSN 0x3ff801f +#define PARISC_KPROBES_BREAK_INSN2 0x3ff801e #define __ARCH_WANT_KPROBES_INSN_SLOT -#define MAX_INSN_SIZE 1 +#define MAX_INSN_SIZE 2 typedef u32 kprobe_opcode_t; struct kprobe; @@ -29,7 +30,7 @@ void arch_remove_kprobe(struct kprobe *p); #define flush_insn_slot(p) \ flush_icache_range((unsigned long)&(p)->ainsn.insn[0], \ (unsigned long)&(p)->ainsn.insn[0] + \ - sizeof(kprobe_opcode_t)) + MAX_INSN_SIZE*sizeof(kprobe_opcode_t)) #define kretprobe_blacklist_size 0 diff --git a/arch/parisc/include/asm/mmu.h b/arch/parisc/include/asm/mmu.h index 3fb70a601d5c..44fd062b62ed 100644 --- a/arch/parisc/include/asm/mmu.h +++ b/arch/parisc/include/asm/mmu.h @@ -2,7 +2,9 @@ #ifndef _PARISC_MMU_H_ #define _PARISC_MMU_H_ -/* On parisc, we store the space id here */ -typedef unsigned long mm_context_t; +typedef struct { + unsigned long space_id; + unsigned long vdso_base; +} mm_context_t; #endif /* _PARISC_MMU_H_ */ diff --git a/arch/parisc/include/asm/mmu_context.h b/arch/parisc/include/asm/mmu_context.h index 726257648d9f..c9187fe836a3 100644 --- a/arch/parisc/include/asm/mmu_context.h +++ b/arch/parisc/include/asm/mmu_context.h @@ -20,7 +20,7 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { BUG_ON(atomic_read(&mm->mm_users) != 1); - mm->context = alloc_sid(); + mm->context.space_id = alloc_sid(); return 0; } @@ -28,22 +28,22 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) static inline void destroy_context(struct mm_struct *mm) { - free_sid(mm->context); - mm->context = 0; + free_sid(mm->context.space_id); + mm->context.space_id = 0; } static inline unsigned long __space_to_prot(mm_context_t context) { #if SPACEID_SHIFT == 0 - return context << 1; + return context.space_id << 1; #else - return context >> (SPACEID_SHIFT - 1); + return context.space_id >> (SPACEID_SHIFT - 1); #endif } static inline void load_context(mm_context_t context) { - mtsp(context, 3); + mtsp(context.space_id, SR_USER); mtctl(__space_to_prot(context), 8); } @@ -89,8 +89,8 @@ static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) BUG_ON(next == &init_mm); /* Should never happen */ - if (next->context == 0) - next->context = alloc_sid(); + if (next->context.space_id == 0) + next->context.space_id = alloc_sid(); switch_mm(prev,next,current); } diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 3e7cf882639f..939db6fe620b 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -70,9 +70,9 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) unsigned long flags; purge_tlb_start(flags); - mtsp(mm->context, 1); - pdtlb(addr); - pitlb(addr); + mtsp(mm->context.space_id, SR_TEMP1); + pdtlb(SR_TEMP1, addr); + pitlb(SR_TEMP1, addr); purge_tlb_end(flags); } @@ -219,9 +219,10 @@ extern void __update_cache(pte_t pte); #define _PAGE_PRESENT (1 << xlate_pabit(_PAGE_PRESENT_BIT)) #define _PAGE_HUGE (1 << xlate_pabit(_PAGE_HPAGE_BIT)) #define _PAGE_USER (1 << xlate_pabit(_PAGE_USER_BIT)) +#define _PAGE_SPECIAL (_PAGE_DMB) #define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED) -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY) +#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_SPECIAL) #define _PAGE_KERNEL_RO (_PAGE_PRESENT | _PAGE_READ | _PAGE_DIRTY | _PAGE_ACCESSED) #define _PAGE_KERNEL_EXEC (_PAGE_KERNEL_RO | _PAGE_EXEC) #define _PAGE_KERNEL_RWX (_PAGE_KERNEL_EXEC | _PAGE_WRITE) @@ -348,6 +349,7 @@ static inline void pud_clear(pud_t *pud) { static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } +static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~_PAGE_DIRTY; return pte; } static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; return pte; } @@ -355,6 +357,7 @@ static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~_PAGE_WRITE; ret static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; } static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; } static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; } +static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; } /* * Huge pte definitions. @@ -405,6 +408,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return ((unsigned long) __va(pmd_address(pmd))); } +#define pmd_pfn(pmd) (pmd_address(pmd) >> PAGE_SHIFT) #define __pmd_page(pmd) ((unsigned long) __va(pmd_address(pmd))) #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h index 3a3d05438408..006364212795 100644 --- a/arch/parisc/include/asm/processor.h +++ b/arch/parisc/include/asm/processor.h @@ -236,7 +236,7 @@ on downward growing arches, it looks like this: #define start_thread(regs, new_pc, new_sp) do { \ elf_addr_t *sp = (elf_addr_t *)new_sp; \ - __u32 spaceid = (__u32)current->mm->context; \ + __u32 spaceid = (__u32)current->mm->context.space_id; \ elf_addr_t pc = (elf_addr_t)new_pc | 3; \ elf_caddr_t *argv = (elf_caddr_t *)bprm->exec + 1; \ \ diff --git a/arch/parisc/include/asm/rt_sigframe.h b/arch/parisc/include/asm/rt_sigframe.h index 2b3010ade00e..bb7fb4153327 100644 --- a/arch/parisc/include/asm/rt_sigframe.h +++ b/arch/parisc/include/asm/rt_sigframe.h @@ -2,16 +2,8 @@ #ifndef _ASM_PARISC_RT_SIGFRAME_H #define _ASM_PARISC_RT_SIGFRAME_H -#define SIGRETURN_TRAMP 4 -#define SIGRESTARTBLOCK_TRAMP 5 -#define TRAMP_SIZE (SIGRETURN_TRAMP + SIGRESTARTBLOCK_TRAMP) - struct rt_sigframe { - /* XXX: Must match trampoline size in arch/parisc/kernel/signal.c - Secondary to that it must protect the ERESTART_RESTARTBLOCK - trampoline we left on the stack (we were bad and didn't - change sp so we could run really fast.) */ - unsigned int tramp[TRAMP_SIZE]; + unsigned int tramp[2]; /* holds original return address */ struct siginfo info; struct ucontext uc; }; diff --git a/arch/parisc/include/asm/special_insns.h b/arch/parisc/include/asm/special_insns.h index 16ee41e77174..41b3ddbd344c 100644 --- a/arch/parisc/include/asm/special_insns.h +++ b/arch/parisc/include/asm/special_insns.h @@ -55,8 +55,8 @@ static inline void set_eiem(unsigned long val) #define mfsp(reg) ({ \ unsigned long cr; \ __asm__ __volatile__( \ - "mfsp " #reg ",%0" : \ - "=r" (cr) \ + "mfsp %%sr%1,%0" \ + : "=r" (cr) : "i"(reg) \ ); \ cr; \ }) diff --git a/arch/parisc/include/asm/tlbflush.h b/arch/parisc/include/asm/tlbflush.h index c5ded01d45be..5ffd7c17f593 100644 --- a/arch/parisc/include/asm/tlbflush.h +++ b/arch/parisc/include/asm/tlbflush.h @@ -17,7 +17,7 @@ int __flush_tlb_range(unsigned long sid, unsigned long start, unsigned long end); #define flush_tlb_range(vma, start, end) \ - __flush_tlb_range((vma)->vm_mm->context, start, end) + __flush_tlb_range((vma)->vm_mm->context.space_id, start, end) #define flush_tlb_kernel_range(start, end) \ __flush_tlb_range(0, start, end) diff --git a/arch/parisc/include/asm/traps.h b/arch/parisc/include/asm/traps.h index 34619f010c63..0ccdb738a9a3 100644 --- a/arch/parisc/include/asm/traps.h +++ b/arch/parisc/include/asm/traps.h @@ -18,6 +18,7 @@ unsigned long parisc_acctyp(unsigned long code, unsigned int inst); const char *trap_name(unsigned long code); void do_page_fault(struct pt_regs *regs, unsigned long code, unsigned long address); +int handle_nadtlb_fault(struct pt_regs *regs); #endif #endif diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index ebf8a845b017..2bf660eabe42 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -11,15 +11,9 @@ #include <linux/bug.h> #include <linux/string.h> -/* - * Note that since kernel addresses are in a separate address space on - * parisc, we don't need to do anything for access_ok(). - * We just let the page fault handler do the right thing. This also means - * that put_user is the same as __put_user, etc. - */ - -#define access_ok(uaddr, size) \ - ( (uaddr) == (uaddr) ) +#define TASK_SIZE_MAX DEFAULT_TASK_SIZE +#include <asm/pgtable.h> +#include <asm-generic/access_ok.h> #define put_user __put_user #define get_user __get_user @@ -79,28 +73,27 @@ struct exception_table_entry { #define __get_user(val, ptr) \ ({ \ - __get_user_internal("%%sr3,", val, ptr); \ + __get_user_internal(SR_USER, val, ptr); \ }) #define __get_user_asm(sr, val, ldx, ptr) \ { \ register long __gu_val; \ \ - __asm__("1: " ldx " 0(" sr "%2),%0\n" \ + __asm__("1: " ldx " 0(%%sr%2,%3),%0\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ - : "=r"(__gu_val), "=r"(__gu_err) \ - : "r"(ptr), "1"(__gu_err)); \ + : "=r"(__gu_val), "+r"(__gu_err) \ + : "i"(sr), "r"(ptr)); \ \ (val) = (__force __typeof__(*(ptr))) __gu_val; \ } -#define HAVE_GET_KERNEL_NOFAULT #define __get_kernel_nofault(dst, src, type, err_label) \ { \ type __z; \ long __err; \ - __err = __get_user_internal("%%sr0,", __z, (type *)(src)); \ + __err = __get_user_internal(SR_KERNEL, __z, (type *)(src)); \ if (unlikely(__err)) \ goto err_label; \ else \ @@ -118,13 +111,13 @@ struct exception_table_entry { } __gu_tmp; \ \ __asm__(" copy %%r0,%R0\n" \ - "1: ldw 0(" sr "%2),%0\n" \ - "2: ldw 4(" sr "%2),%R0\n" \ + "1: ldw 0(%%sr%2,%3),%0\n" \ + "2: ldw 4(%%sr%2,%3),%R0\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b) \ - : "=&r"(__gu_tmp.l), "=r"(__gu_err) \ - : "r"(ptr), "1"(__gu_err)); \ + : "=&r"(__gu_tmp.l), "+r"(__gu_err) \ + : "i"(sr), "r"(ptr)); \ \ (val) = __gu_tmp.t; \ } @@ -135,13 +128,12 @@ struct exception_table_entry { #define __put_user_internal(sr, x, ptr) \ ({ \ ASM_EXCEPTIONTABLE_VAR(__pu_err); \ - __typeof__(*(ptr)) __x = (__typeof__(*(ptr)))(x); \ \ switch (sizeof(*(ptr))) { \ - case 1: __put_user_asm(sr, "stb", __x, ptr); break; \ - case 2: __put_user_asm(sr, "sth", __x, ptr); break; \ - case 4: __put_user_asm(sr, "stw", __x, ptr); break; \ - case 8: STD_USER(sr, __x, ptr); break; \ + case 1: __put_user_asm(sr, "stb", x, ptr); break; \ + case 2: __put_user_asm(sr, "sth", x, ptr); break; \ + case 4: __put_user_asm(sr, "stw", x, ptr); break; \ + case 8: STD_USER(sr, x, ptr); break; \ default: BUILD_BUG(); \ } \ \ @@ -150,14 +142,16 @@ struct exception_table_entry { #define __put_user(x, ptr) \ ({ \ - __put_user_internal("%%sr3,", x, ptr); \ + __typeof__(&*(ptr)) __ptr = ptr; \ + __typeof__(*(__ptr)) __x = (__typeof__(*(__ptr)))(x); \ + __put_user_internal(SR_USER, __x, __ptr); \ }) #define __put_kernel_nofault(dst, src, type, err_label) \ { \ type __z = *(type *)(src); \ long __err; \ - __err = __put_user_internal("%%sr0,", __z, (type *)(dst)); \ + __err = __put_user_internal(SR_KERNEL, __z, (type *)(dst)); \ if (unlikely(__err)) \ goto err_label; \ } @@ -177,24 +171,24 @@ struct exception_table_entry { #define __put_user_asm(sr, stx, x, ptr) \ __asm__ __volatile__ ( \ - "1: " stx " %2,0(" sr "%1)\n" \ + "1: " stx " %1,0(%%sr%2,%3)\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ - : "=r"(__pu_err) \ - : "r"(ptr), "r"(x), "0"(__pu_err)) + : "+r"(__pu_err) \ + : "r"(x), "i"(sr), "r"(ptr)) #if !defined(CONFIG_64BIT) #define __put_user_asm64(sr, __val, ptr) do { \ __asm__ __volatile__ ( \ - "1: stw %2,0(" sr "%1)\n" \ - "2: stw %R2,4(" sr "%1)\n" \ + "1: stw %1,0(%%sr%2,%3)\n" \ + "2: stw %R1,4(%%sr%2,%3)\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b) \ - : "=r"(__pu_err) \ - : "r"(ptr), "r"(__val), "0"(__pu_err)); \ + : "+r"(__pu_err) \ + : "r"(__val), "i"(sr), "r"(ptr)); \ } while (0) #endif /* !defined(CONFIG_64BIT) */ diff --git a/arch/parisc/include/asm/unistd.h b/arch/parisc/include/asm/unistd.h index cd438e4150f6..7708a5806f09 100644 --- a/arch/parisc/include/asm/unistd.h +++ b/arch/parisc/include/asm/unistd.h @@ -63,10 +63,6 @@ ); \ __sys_res = (long)__res; \ } \ - if ( (unsigned long)__sys_res >= (unsigned long)-4095 ){ \ - errno = -__sys_res; \ - __sys_res = -1; \ - } \ __sys_res; \ }) diff --git a/arch/parisc/include/asm/vdso.h b/arch/parisc/include/asm/vdso.h new file mode 100644 index 000000000000..ef8206193f82 --- /dev/null +++ b/arch/parisc/include/asm/vdso.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PARISC_VDSO_H__ +#define __PARISC_VDSO_H__ + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_64BIT +#include <generated/vdso64-offsets.h> +#endif +#include <generated/vdso32-offsets.h> + +#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name)) +#define VDSO32_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso32_offset_##name)) + +extern struct vdso_data *vdso_data; + +#endif /* __ASSEMBLY __ */ + +/* Default link addresses for the vDSOs */ +#define VDSO_LBASE 0 + +#define VDSO_VERSION_STRING LINUX_5.18 + +#endif /* __PARISC_VDSO_H__ */ diff --git a/arch/parisc/include/uapi/asm/auxvec.h b/arch/parisc/include/uapi/asm/auxvec.h new file mode 100644 index 000000000000..90d2aa699cf3 --- /dev/null +++ b/arch/parisc/include/uapi/asm/auxvec.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PARISC_AUXVEC_H +#define _UAPI_PARISC_AUXVEC_H + +/* The vDSO location. */ +#define AT_SYSINFO_EHDR 33 + +#endif /* _UAPI_PARISC_AUXVEC_H */ diff --git a/arch/parisc/include/uapi/asm/shmbuf.h b/arch/parisc/include/uapi/asm/shmbuf.h index 5da3089be65e..532da742fb56 100644 --- a/arch/parisc/include/uapi/asm/shmbuf.h +++ b/arch/parisc/include/uapi/asm/shmbuf.h @@ -3,6 +3,8 @@ #define _PARISC_SHMBUF_H #include <asm/bitsperlong.h> +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> /* * The shmid64_ds structure for parisc architecture. diff --git a/arch/parisc/include/uapi/asm/signal.h b/arch/parisc/include/uapi/asm/signal.h index e5a2657477ac..8e4895c5ea5d 100644 --- a/arch/parisc/include/uapi/asm/signal.h +++ b/arch/parisc/include/uapi/asm/signal.h @@ -67,7 +67,7 @@ struct siginfo; typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; #endif /* !__ASSEMBLY */ diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile index 8fb819bbbb17..d579243edc2f 100644 --- a/arch/parisc/kernel/Makefile +++ b/arch/parisc/kernel/Makefile @@ -39,3 +39,8 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KEXEC_CORE) += kexec.o relocate_kernel.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o + +# vdso +obj-y += vdso.o +obj-$(CONFIG_64BIT) += vdso64/ +obj-y += vdso32/ diff --git a/arch/parisc/kernel/alternative.c b/arch/parisc/kernel/alternative.c index fa28c4c9f972..daa1e9047275 100644 --- a/arch/parisc/kernel/alternative.c +++ b/arch/parisc/kernel/alternative.c @@ -8,6 +8,7 @@ #include <asm/processor.h> #include <asm/sections.h> #include <asm/alternative.h> +#include <asm/cacheflush.h> #include <linux/module.h> @@ -107,5 +108,14 @@ void __init apply_alternatives_all(void) apply_alternatives((struct alt_instr *) &__alt_instructions, (struct alt_instr *) &__alt_instructions_end, NULL); + if (cache_info.dc_size == 0 && cache_info.ic_size == 0) { + pr_info("alternatives: optimizing cache-flushes.\n"); + static_branch_disable(&parisc_has_cache); + } + if (cache_info.dc_size == 0) + static_branch_disable(&parisc_has_dcache); + if (cache_info.ic_size == 0) + static_branch_disable(&parisc_has_icache); + set_kernel_text_rw(0); } diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index 2a83ef36d216..2673d57eeb00 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -26,7 +26,11 @@ #include <asm/ptrace.h> #include <asm/processor.h> #include <asm/pdc.h> +#include <uapi/asm/sigcontext.h> +#include <asm/ucontext.h> +#include <asm/rt_sigframe.h> #include <linux/uaccess.h> +#include "signal32.h" /* Add FRAME_SIZE to the size x and align it to y. All definitions * that use align_frame will include space for a frame. @@ -218,6 +222,11 @@ int main(void) DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_PRE_COUNT, offsetof(struct task_struct, thread_info.preempt_count)); BLANK(); + DEFINE(ASM_SIGFRAME_SIZE, PARISC_RT_SIGFRAME_SIZE); + DEFINE(SIGFRAME_CONTEXT_REGS, offsetof(struct rt_sigframe, uc.uc_mcontext) - PARISC_RT_SIGFRAME_SIZE); + DEFINE(ASM_SIGFRAME_SIZE32, PARISC_RT_SIGFRAME_SIZE32); + DEFINE(SIGFRAME_CONTEXT_REGS32, offsetof(struct compat_rt_sigframe, uc.uc_mcontext) - PARISC_RT_SIGFRAME_SIZE32); + BLANK(); DEFINE(ICACHE_BASE, offsetof(struct pdc_cache_info, ic_base)); DEFINE(ICACHE_STRIDE, offsetof(struct pdc_cache_info, ic_stride)); DEFINE(ICACHE_COUNT, offsetof(struct pdc_cache_info, ic_count)); diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 94150b91c96f..456e879d34a8 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -38,6 +38,9 @@ EXPORT_SYMBOL(flush_dcache_page_asm); void purge_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr); void flush_icache_page_asm(unsigned long phys_addr, unsigned long vaddr); +/* Internal implementation in arch/parisc/kernel/pacache.S */ +void flush_data_cache_local(void *); /* flushes local data-cache only */ +void flush_instruction_cache_local(void); /* flushes local code-cache only */ /* On some machines (i.e., ones with the Merced bus), there can be * only a single PxTLB broadcast at a time; this must be guaranteed @@ -58,26 +61,35 @@ struct pdc_cache_info cache_info __ro_after_init; static struct pdc_btlb_info btlb_info __ro_after_init; #endif -#ifdef CONFIG_SMP -void -flush_data_cache(void) +DEFINE_STATIC_KEY_TRUE(parisc_has_cache); +DEFINE_STATIC_KEY_TRUE(parisc_has_dcache); +DEFINE_STATIC_KEY_TRUE(parisc_has_icache); + +static void cache_flush_local_cpu(void *dummy) { - on_each_cpu(flush_data_cache_local, NULL, 1); + if (static_branch_likely(&parisc_has_icache)) + flush_instruction_cache_local(); + if (static_branch_likely(&parisc_has_dcache)) + flush_data_cache_local(NULL); } -void -flush_instruction_cache(void) + +void flush_cache_all_local(void) { - on_each_cpu(flush_instruction_cache_local, NULL, 1); + cache_flush_local_cpu(NULL); } -#endif -void -flush_cache_all_local(void) +void flush_cache_all(void) { - flush_instruction_cache_local(NULL); - flush_data_cache_local(NULL); + if (static_branch_likely(&parisc_has_cache)) + on_each_cpu(cache_flush_local_cpu, NULL, 1); } -EXPORT_SYMBOL(flush_cache_all_local); + +static inline void flush_data_cache(void) +{ + if (static_branch_likely(&parisc_has_dcache)) + on_each_cpu(flush_data_cache_local, NULL, 1); +} + /* Virtual address of pfn. */ #define pfn_va(pfn) __va(PFN_PHYS(pfn)) @@ -303,6 +315,8 @@ static inline void __flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long physaddr) { + if (!static_branch_likely(&parisc_has_cache)) + return; preempt_disable(); flush_dcache_page_asm(physaddr, vmaddr); if (vma->vm_flags & VM_EXEC) @@ -314,6 +328,8 @@ static inline void __purge_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long physaddr) { + if (!static_branch_likely(&parisc_has_cache)) + return; preempt_disable(); purge_dcache_page_asm(physaddr, vmaddr); if (vma->vm_flags & VM_EXEC) @@ -375,7 +391,6 @@ EXPORT_SYMBOL(flush_dcache_page); /* Defined in arch/parisc/kernel/pacache.S */ EXPORT_SYMBOL(flush_kernel_dcache_range_asm); -EXPORT_SYMBOL(flush_data_cache_local); EXPORT_SYMBOL(flush_kernel_icache_range_asm); #define FLUSH_THRESHOLD 0x80000 /* 0.5MB */ @@ -388,7 +403,7 @@ void __init parisc_setup_cache_timing(void) { unsigned long rangetime, alltime; unsigned long size; - unsigned long threshold; + unsigned long threshold, threshold2; alltime = mfctl(16); flush_data_cache(); @@ -403,8 +418,20 @@ void __init parisc_setup_cache_timing(void) alltime, size, rangetime); threshold = L1_CACHE_ALIGN(size * alltime / rangetime); - if (threshold > cache_info.dc_size) - threshold = cache_info.dc_size; + + /* + * The threshold computed above isn't very reliable since the + * flush times depend greatly on the percentage of dirty lines + * in the flush range. Further, the whole cache time doesn't + * include the time to refill lines that aren't in the mm/vma + * being flushed. By timing glibc build and checks on mako cpus, + * the following formula seems to work reasonably well. The + * value from the timing calculation is too small, and increases + * build and check times by almost a factor two. + */ + threshold2 = cache_info.dc_size * num_online_cpus(); + if (threshold2 > threshold) + threshold = threshold2; if (threshold) parisc_cache_flush_threshold = threshold; printk(KERN_INFO "Cache flush threshold set to %lu KiB\n", @@ -457,7 +484,7 @@ void flush_kernel_dcache_page_addr(void *addr) flush_kernel_dcache_page_asm(addr); purge_tlb_start(flags); - pdtlb_kernel(addr); + pdtlb(SR_KERNEL, addr); purge_tlb_end(flags); } EXPORT_SYMBOL(flush_kernel_dcache_page_addr); @@ -496,25 +523,15 @@ int __flush_tlb_range(unsigned long sid, unsigned long start, but cause a purge request to be broadcast to other TLBs. */ while (start < end) { purge_tlb_start(flags); - mtsp(sid, 1); - pdtlb(start); - pitlb(start); + mtsp(sid, SR_TEMP1); + pdtlb(SR_TEMP1, start); + pitlb(SR_TEMP1, start); purge_tlb_end(flags); start += PAGE_SIZE; } return 0; } -static void cacheflush_h_tmp_function(void *dummy) -{ - flush_cache_all_local(); -} - -void flush_cache_all(void) -{ - on_each_cpu(cacheflush_h_tmp_function, NULL, 1); -} - static inline unsigned long mm_total_size(struct mm_struct *mm) { struct vm_area_struct *vma; @@ -558,15 +575,6 @@ static void flush_cache_pages(struct vm_area_struct *vma, struct mm_struct *mm, } } -static void flush_user_cache_tlb(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - flush_user_dcache_range_asm(start, end); - if (vma->vm_flags & VM_EXEC) - flush_user_icache_range_asm(start, end); - flush_tlb_range(vma, start, end); -} - void flush_cache_mm(struct mm_struct *mm) { struct vm_area_struct *vma; @@ -575,23 +583,14 @@ void flush_cache_mm(struct mm_struct *mm) rp3440, etc. So, avoid it if the mm isn't too big. */ if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && mm_total_size(mm) >= parisc_cache_flush_threshold) { - if (mm->context) + if (mm->context.space_id) flush_tlb_all(); flush_cache_all(); return; } - preempt_disable(); - if (mm->context == mfsp(3)) { - for (vma = mm->mmap; vma; vma = vma->vm_next) - flush_user_cache_tlb(vma, vma->vm_start, vma->vm_end); - preempt_enable(); - return; - } - for (vma = mm->mmap; vma; vma = vma->vm_next) flush_cache_pages(vma, mm, vma->vm_start, vma->vm_end); - preempt_enable(); } void flush_cache_range(struct vm_area_struct *vma, @@ -599,29 +598,21 @@ void flush_cache_range(struct vm_area_struct *vma, { if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && end - start >= parisc_cache_flush_threshold) { - if (vma->vm_mm->context) + if (vma->vm_mm->context.space_id) flush_tlb_range(vma, start, end); flush_cache_all(); return; } - preempt_disable(); - if (vma->vm_mm->context == mfsp(3)) { - flush_user_cache_tlb(vma, start, end); - preempt_enable(); - return; - } - - flush_cache_pages(vma, vma->vm_mm, vma->vm_start, vma->vm_end); - preempt_enable(); + flush_cache_pages(vma, vma->vm_mm, start, end); } void flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn) { if (pfn_valid(pfn)) { - if (likely(vma->vm_mm->context)) { - flush_tlb_page(vma, vmaddr); + flush_tlb_page(vma, vmaddr); + if (likely(vma->vm_mm->context.space_id)) { __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn)); } else { __purge_cache_page(vma, vmaddr, PFN_PHYS(pfn)); @@ -633,6 +624,7 @@ void flush_kernel_vmap_range(void *vaddr, int size) { unsigned long start = (unsigned long)vaddr; unsigned long end = start + size; + unsigned long flags, physaddr; if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && (unsigned long)size >= parisc_cache_flush_threshold) { @@ -641,8 +633,14 @@ void flush_kernel_vmap_range(void *vaddr, int size) return; } - flush_kernel_dcache_range_asm(start, end); - flush_tlb_kernel_range(start, end); + while (start < end) { + physaddr = lpa(start); + purge_tlb_start(flags); + pdtlb(SR_KERNEL, start); + purge_tlb_end(flags); + flush_dcache_page_asm(physaddr, start); + start += PAGE_SIZE; + } } EXPORT_SYMBOL(flush_kernel_vmap_range); @@ -650,6 +648,7 @@ void invalidate_kernel_vmap_range(void *vaddr, int size) { unsigned long start = (unsigned long)vaddr; unsigned long end = start + size; + unsigned long flags, physaddr; if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && (unsigned long)size >= parisc_cache_flush_threshold) { @@ -658,7 +657,13 @@ void invalidate_kernel_vmap_range(void *vaddr, int size) return; } - purge_kernel_dcache_range_asm(start, end); - flush_tlb_kernel_range(start, end); + while (start < end) { + physaddr = lpa(start); + purge_tlb_start(flags); + pdtlb(SR_KERNEL, start); + purge_tlb_end(flags); + purge_dcache_page_asm(physaddr, start); + start += PAGE_SIZE; + } } EXPORT_SYMBOL(invalidate_kernel_vmap_range); diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index 6e9cdb269862..ecf50159359e 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -1288,74 +1288,12 @@ nadtlb_check_alias_20: nadtlb_emulate: /* - * Non access misses can be caused by fdc,fic,pdc,lpa,probe and - * probei instructions. We don't want to fault for these - * instructions (not only does it not make sense, it can cause - * deadlocks, since some flushes are done with the mmap - * semaphore held). If the translation doesn't exist, we can't - * insert a translation, so have to emulate the side effects - * of the instruction. Since we don't insert a translation - * we can get a lot of faults during a flush loop, so it makes - * sense to try to do it here with minimum overhead. We only - * emulate fdc,fic,pdc,probew,prober instructions whose base - * and index registers are not shadowed. We defer everything - * else to the "slow" path. + * Non-access misses can be caused by fdc,fic,pdc,lpa,probe and + * probei instructions. The kernel no longer faults doing flushes. + * Use of lpa and probe instructions is rare. Given the issue + * with shadow registers, we defer everything to the "slow" path. */ - - mfctl %cr19,%r9 /* Get iir */ - - /* PA 2.0 Arch Ref. Book pg 382 has a good description of the insn bits. - Checks for fdc,fdce,pdc,"fic,4f",prober,probeir,probew, probeiw */ - - /* Checks for fdc,fdce,pdc,"fic,4f" only */ - ldi 0x280,%r16 - and %r9,%r16,%r17 - cmpb,<>,n %r16,%r17,nadtlb_probe_check - bb,>=,n %r9,26,nadtlb_nullify /* m bit not set, just nullify */ - BL get_register,%r25 - extrw,u %r9,15,5,%r8 /* Get index register # */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - copy %r1,%r24 - BL get_register,%r25 - extrw,u %r9,10,5,%r8 /* Get base register # */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - BL set_register,%r25 - add,l %r1,%r24,%r1 /* doesn't affect c/b bits */ - -nadtlb_nullify: - mfctl %ipsw,%r8 - ldil L%PSW_N,%r9 - or %r8,%r9,%r8 /* Set PSW_N */ - mtctl %r8,%ipsw - - rfir - nop - - /* - When there is no translation for the probe address then we - must nullify the insn and return zero in the target register. - This will indicate to the calling code that it does not have - write/read privileges to this address. - - This should technically work for prober and probew in PA 1.1, - and also probe,r and probe,w in PA 2.0 - - WARNING: USE ONLY NON-SHADOW REGISTERS WITH PROBE INSN! - THE SLOW-PATH EMULATION HAS NOT BEEN WRITTEN YET. - - */ -nadtlb_probe_check: - ldi 0x80,%r16 - and %r9,%r16,%r17 - cmpb,<>,n %r16,%r17,nadtlb_fault /* Must be probe,[rw]*/ - BL get_register,%r25 /* Find the target register */ - extrw,u %r9,31,5,%r8 /* Get target register */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - BL set_register,%r25 - copy %r0,%r1 /* Write zero to target register */ - b nadtlb_nullify /* Nullify return insn */ - nop - + b,n nadtlb_fault #ifdef CONFIG_64BIT itlb_miss_20w: diff --git a/arch/parisc/kernel/kprobes.c b/arch/parisc/kernel/kprobes.c index e2bdb5a5f93e..3343d2fb7889 100644 --- a/arch/parisc/kernel/kprobes.c +++ b/arch/parisc/kernel/kprobes.c @@ -5,6 +5,7 @@ * PA-RISC kprobes implementation * * Copyright (c) 2019 Sven Schnelle <svens@stackframe.org> + * Copyright (c) 2022 Helge Deller <deller@gmx.de> */ #include <linux/types.h> @@ -25,9 +26,14 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if (!p->ainsn.insn) return -ENOMEM; - memcpy(p->ainsn.insn, p->addr, - MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); + /* + * Set up new instructions. Second break instruction will + * trigger call of parisc_kprobe_ss_handler(). + */ p->opcode = *p->addr; + p->ainsn.insn[0] = p->opcode; + p->ainsn.insn[1] = PARISC_KPROBES_BREAK_INSN2; + flush_insn_slot(p); return 0; } @@ -73,9 +79,7 @@ static void __kprobes setup_singlestep(struct kprobe *p, { kcb->iaoq[0] = regs->iaoq[0]; kcb->iaoq[1] = regs->iaoq[1]; - regs->iaoq[0] = (unsigned long)p->ainsn.insn; - mtctl(0, 0); - regs->gr[0] |= PSW_R; + instruction_pointer_set(regs, (unsigned long)p->ainsn.insn); } int __kprobes parisc_kprobe_break_handler(struct pt_regs *regs) @@ -165,9 +169,8 @@ int __kprobes parisc_kprobe_ss_handler(struct pt_regs *regs) regs->iaoq[0] = kcb->iaoq[1]; break; default: - regs->iaoq[1] = kcb->iaoq[0]; - regs->iaoq[1] += (regs->iaoq[1] - regs->iaoq[0]) + 4; regs->iaoq[0] = kcb->iaoq[1]; + regs->iaoq[1] = regs->iaoq[0] + 4; break; } kcb->kprobe_status = KPROBE_HIT_SSDONE; @@ -191,14 +194,17 @@ static struct kprobe trampoline_p = { static int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - unsigned long orig_ret_address; - - orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); - instruction_pointer_set(regs, orig_ret_address); + __kretprobe_trampoline_handler(regs, NULL); return 1; } +void arch_kretprobe_fixup_return(struct pt_regs *regs, + kprobe_opcode_t *correct_ret_addr) +{ + regs->gr[2] = (unsigned long)correct_ret_addr; +} + void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index 36a57aa38e87..160996f2198e 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -91,7 +91,7 @@ static inline int map_pte_uncached(pte_t * pte, printk(KERN_ERR "map_pte_uncached: page already exists\n"); purge_tlb_start(flags); set_pte(pte, __mk_pte(*paddr_ptr, PAGE_KERNEL_UNC)); - pdtlb_kernel(orig_vaddr); + pdtlb(SR_KERNEL, orig_vaddr); purge_tlb_end(flags); vaddr += PAGE_SIZE; orig_vaddr += PAGE_SIZE; @@ -175,7 +175,7 @@ static inline void unmap_uncached_pte(pmd_t * pmd, unsigned long vaddr, pte_clear(&init_mm, vaddr, pte); purge_tlb_start(flags); - pdtlb_kernel(orig_vaddr); + pdtlb(SR_KERNEL, orig_vaddr); purge_tlb_end(flags); vaddr += PAGE_SIZE; orig_vaddr += PAGE_SIZE; diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c index 46b1050640b8..3dd672e2e301 100644 --- a/arch/parisc/kernel/signal.c +++ b/arch/parisc/kernel/signal.c @@ -1,16 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 /* - * linux/arch/parisc/kernel/signal.c: Architecture-specific signal - * handling support. + * PA-RISC architecture-specific signal handling support. * * Copyright (C) 2000 David Huggins-Daines <dhd@debian.org> * Copyright (C) 2000 Linuxcare, Inc. + * Copyright (C) 2000-2022 Helge Deller <deller@gmx.de> + * Copyright (C) 2022 John David Anglin <dave.anglin@bell.net> * * Based on the ia64, i386, and alpha versions. - * - * Like the IA-64, we are a recent enough port (we are *starting* - * with glibc2.2) that we do not need to support the old non-realtime - * Linux signals. Therefore we don't. */ #include <linux/sched.h> @@ -32,6 +29,7 @@ #include <linux/uaccess.h> #include <asm/cacheflush.h> #include <asm/asm-offsets.h> +#include <asm/vdso.h> #ifdef CONFIG_COMPAT #include "signal32.h" @@ -59,14 +57,6 @@ * Do a signal return - restore sigcontext. */ -/* Trampoline for calling rt_sigreturn() */ -#define INSN_LDI_R25_0 0x34190000 /* ldi 0,%r25 (in_syscall=0) */ -#define INSN_LDI_R25_1 0x34190002 /* ldi 1,%r25 (in_syscall=1) */ -#define INSN_LDI_R20 0x3414015a /* ldi __NR_rt_sigreturn,%r20 */ -#define INSN_BLE_SR2_R0 0xe4008200 /* be,l 0x100(%sr2,%r0),%sr0,%r31 */ -/* For debugging */ -#define INSN_DIE_HORRIBLY 0x68000ccc /* stw %r0,0x666(%sr0,%r0) */ - static long restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) { @@ -77,9 +67,9 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) err |= __copy_from_user(regs->iaoq, sc->sc_iaoq, sizeof(regs->iaoq)); err |= __copy_from_user(regs->iasq, sc->sc_iasq, sizeof(regs->iasq)); err |= __get_user(regs->sar, &sc->sc_sar); - DBG(2,"restore_sigcontext: iaoq is %#lx / %#lx\n", - regs->iaoq[0],regs->iaoq[1]); - DBG(2,"restore_sigcontext: r28 is %ld\n", regs->gr[28]); + DBG(2, "%s: iaoq is %#lx / %#lx\n", + __func__, regs->iaoq[0], regs->iaoq[1]); + DBG(2, "%s: r28 is %ld\n", __func__, regs->gr[28]); return err; } @@ -102,7 +92,7 @@ sys_rt_sigreturn(struct pt_regs *regs, int in_syscall) /* Unwind the user stack to get the rt_sigframe structure. */ frame = (struct rt_sigframe __user *) (usp - sigframe_size); - DBG(2,"sys_rt_sigreturn: frame is %p\n", frame); + DBG(2, "%s: frame is %p pid %d\n", __func__, frame, task_pid_nr(current)); regs->orig_r28 = 1; /* no restarts for sigreturn */ @@ -110,7 +100,6 @@ sys_rt_sigreturn(struct pt_regs *regs, int in_syscall) compat_frame = (struct compat_rt_sigframe __user *)frame; if (is_compat_task()) { - DBG(2,"sys_rt_sigreturn: ELF32 process.\n"); if (get_compat_sigset(&set, &compat_frame->uc.uc_sigmask)) goto give_sigsegv; } else @@ -125,25 +114,25 @@ sys_rt_sigreturn(struct pt_regs *regs, int in_syscall) /* Good thing we saved the old gr[30], eh? */ #ifdef CONFIG_64BIT if (is_compat_task()) { - DBG(1,"sys_rt_sigreturn: compat_frame->uc.uc_mcontext 0x%p\n", - &compat_frame->uc.uc_mcontext); + DBG(1, "%s: compat_frame->uc.uc_mcontext 0x%p\n", + __func__, &compat_frame->uc.uc_mcontext); // FIXME: Load upper half from register file if (restore_sigcontext32(&compat_frame->uc.uc_mcontext, &compat_frame->regs, regs)) goto give_sigsegv; - DBG(1,"sys_rt_sigreturn: usp %#08lx stack 0x%p\n", - usp, &compat_frame->uc.uc_stack); + DBG(1, "%s: usp %#08lx stack 0x%p\n", + __func__, usp, &compat_frame->uc.uc_stack); if (compat_restore_altstack(&compat_frame->uc.uc_stack)) goto give_sigsegv; } else #endif { - DBG(1,"sys_rt_sigreturn: frame->uc.uc_mcontext 0x%p\n", - &frame->uc.uc_mcontext); + DBG(1, "%s: frame->uc.uc_mcontext 0x%p\n", + __func__, &frame->uc.uc_mcontext); if (restore_sigcontext(&frame->uc.uc_mcontext, regs)) goto give_sigsegv; - DBG(1,"sys_rt_sigreturn: usp %#08lx stack 0x%p\n", - usp, &frame->uc.uc_stack); + DBG(1, "%s: usp %#08lx stack 0x%p\n", + __func__, usp, &frame->uc.uc_stack); if (restore_altstack(&frame->uc.uc_stack)) goto give_sigsegv; } @@ -155,14 +144,11 @@ sys_rt_sigreturn(struct pt_regs *regs, int in_syscall) */ if (in_syscall) regs->gr[31] = regs->iaoq[0]; -#if DEBUG_SIG - DBG(1,"sys_rt_sigreturn: returning to %#lx, DUMPING REGS:\n", regs->iaoq[0]); - show_regs(regs); -#endif + return; give_sigsegv: - DBG(1,"sys_rt_sigreturn: Sending SIGSEGV\n"); + DBG(1, "%s: Sending SIGSEGV\n", __func__); force_sig(SIGSEGV); return; } @@ -177,15 +163,15 @@ get_sigframe(struct k_sigaction *ka, unsigned long sp, size_t frame_size) /*FIXME: ELF32 vs. ELF64 has different frame_size, but since we don't use the parameter it doesn't matter */ - DBG(1,"get_sigframe: ka = %#lx, sp = %#lx, frame_size = %#lx\n", - (unsigned long)ka, sp, frame_size); + DBG(1, "%s: ka = %#lx, sp = %#lx, frame_size = %zu\n", + __func__, (unsigned long)ka, sp, frame_size); /* Align alternate stack and reserve 64 bytes for the signal handler's frame marker. */ if ((ka->sa.sa_flags & SA_ONSTACK) != 0 && ! sas_ss_flags(sp)) sp = (current->sas_ss_sp + 0x7f) & ~0x3f; /* Stacks grow up! */ - DBG(1,"get_sigframe: Returning sp = %#lx\n", (unsigned long)sp); + DBG(1, "%s: Returning sp = %#lx\n", __func__, (unsigned long)sp); return (void __user *) sp; /* Stacks grow up. Fun. */ } @@ -205,20 +191,20 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, int in_sysc err |= __put_user(regs->gr[31]+4, &sc->sc_iaoq[1]); err |= __put_user(regs->sr[3], &sc->sc_iasq[0]); err |= __put_user(regs->sr[3], &sc->sc_iasq[1]); - DBG(1,"setup_sigcontext: iaoq %#lx / %#lx (in syscall)\n", - regs->gr[31], regs->gr[31]+4); + DBG(1, "%s: iaoq %#lx / %#lx (in syscall)\n", + __func__, regs->gr[31], regs->gr[31]+4); } else { err |= __copy_to_user(sc->sc_iaoq, regs->iaoq, sizeof(regs->iaoq)); err |= __copy_to_user(sc->sc_iasq, regs->iasq, sizeof(regs->iasq)); - DBG(1,"setup_sigcontext: iaoq %#lx / %#lx (not in syscall)\n", - regs->iaoq[0], regs->iaoq[1]); + DBG(1, "%s: iaoq %#lx / %#lx (not in syscall)\n", + __func__, regs->iaoq[0], regs->iaoq[1]); } err |= __put_user(flags, &sc->sc_flags); err |= __copy_to_user(sc->sc_gr, regs->gr, sizeof(regs->gr)); err |= __copy_to_user(sc->sc_fr, regs->fr, sizeof(regs->fr)); err |= __put_user(regs->sar, &sc->sc_sar); - DBG(1,"setup_sigcontext: r28 is %ld\n", regs->gr[28]); + DBG(1, "%s: r28 is %ld\n", __func__, regs->gr[28]); return err; } @@ -230,7 +216,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, struct rt_sigframe __user *frame; unsigned long rp, usp; unsigned long haddr, sigframe_size; - unsigned long start, end; + unsigned long start; int err = 0; #ifdef CONFIG_64BIT struct compat_rt_sigframe __user * compat_frame; @@ -247,11 +233,10 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, #endif frame = get_sigframe(&ksig->ka, usp, sigframe_size); - DBG(1,"SETUP_RT_FRAME: START\n"); - DBG(1,"setup_rt_frame: frame %p info %p\n", frame, ksig->info); + DBG(1, "%s: frame %p info %p\n", __func__, frame, &ksig->info); start = (unsigned long) frame; - if (start >= user_addr_max() - sigframe_size) + if (start >= TASK_SIZE_MAX - sigframe_size) return -EFAULT; #ifdef CONFIG_64BIT @@ -259,11 +244,12 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, compat_frame = (struct compat_rt_sigframe __user *)frame; if (is_compat_task()) { - DBG(1,"setup_rt_frame: frame->info = 0x%p\n", &compat_frame->info); + DBG(1, "%s: frame->info = 0x%p\n", __func__, &compat_frame->info); err |= copy_siginfo_to_user32(&compat_frame->info, &ksig->info); err |= __compat_save_altstack( &compat_frame->uc.uc_stack, regs->gr[30]); - DBG(1,"setup_rt_frame: frame->uc = 0x%p\n", &compat_frame->uc); - DBG(1,"setup_rt_frame: frame->uc.uc_mcontext = 0x%p\n", &compat_frame->uc.uc_mcontext); + DBG(1, "%s: frame->uc = 0x%p\n", __func__, &compat_frame->uc); + DBG(1, "%s: frame->uc.uc_mcontext = 0x%p\n", + __func__, &compat_frame->uc.uc_mcontext); err |= setup_sigcontext32(&compat_frame->uc.uc_mcontext, &compat_frame->regs, regs, in_syscall); err |= put_compat_sigset(&compat_frame->uc.uc_sigmask, set, @@ -271,11 +257,12 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, } else #endif { - DBG(1,"setup_rt_frame: frame->info = 0x%p\n", &frame->info); + DBG(1, "%s: frame->info = 0x%p\n", __func__, &frame->info); err |= copy_siginfo_to_user(&frame->info, &ksig->info); err |= __save_altstack(&frame->uc.uc_stack, regs->gr[30]); - DBG(1,"setup_rt_frame: frame->uc = 0x%p\n", &frame->uc); - DBG(1,"setup_rt_frame: frame->uc.uc_mcontext = 0x%p\n", &frame->uc.uc_mcontext); + DBG(1, "%s: frame->uc = 0x%p\n", __func__, &frame->uc); + DBG(1, "%s: frame->uc.uc_mcontext = 0x%p\n", + __func__, &frame->uc.uc_mcontext); err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, in_syscall); /* FIXME: Should probably be converted as well for the compat case */ err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); @@ -284,32 +271,15 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, if (err) return -EFAULT; - /* Set up to return from userspace. If provided, use a stub - already in userspace. The first words of tramp are used to - save the previous sigrestartblock trampoline that might be - on the stack. We start the sigreturn trampoline at - SIGRESTARTBLOCK_TRAMP+X. */ - err |= __put_user(in_syscall ? INSN_LDI_R25_1 : INSN_LDI_R25_0, - &frame->tramp[SIGRESTARTBLOCK_TRAMP+0]); - err |= __put_user(INSN_LDI_R20, - &frame->tramp[SIGRESTARTBLOCK_TRAMP+1]); - err |= __put_user(INSN_BLE_SR2_R0, - &frame->tramp[SIGRESTARTBLOCK_TRAMP+2]); - err |= __put_user(INSN_NOP, &frame->tramp[SIGRESTARTBLOCK_TRAMP+3]); - - start = (unsigned long) &frame->tramp[0]; - end = (unsigned long) &frame->tramp[TRAMP_SIZE]; - flush_user_dcache_range_asm(start, end); - flush_user_icache_range_asm(start, end); - - /* TRAMP Words 0-4, Length 5 = SIGRESTARTBLOCK_TRAMP - * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP - * So the SIGRETURN_TRAMP is at the end of SIGRESTARTBLOCK_TRAMP - */ - rp = (unsigned long) &frame->tramp[SIGRESTARTBLOCK_TRAMP]; +#ifdef CONFIG_64BIT + if (!is_compat_task()) + rp = VDSO64_SYMBOL(current, sigtramp_rt); + else +#endif + rp = VDSO32_SYMBOL(current, sigtramp_rt); - if (err) - return -EFAULT; + if (in_syscall) + rp += 4*4; /* skip 4 instructions and start at ldi 1,%r25 */ haddr = A(ksig->ka.sa.sa_handler); /* The sa_handler may be a pointer to a function descriptor */ @@ -340,8 +310,8 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, haddr = fdesc.addr; regs->gr[19] = fdesc.gp; - DBG(1,"setup_rt_frame: 64 bit signal, exe=%#lx, r19=%#lx, in_syscall=%d\n", - haddr, regs->gr[19], in_syscall); + DBG(1, "%s: 64 bit signal, exe=%#lx, r19=%#lx, in_syscall=%d\n", + __func__, haddr, regs->gr[19], in_syscall); } #endif @@ -351,7 +321,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, regs->gr[31] = haddr; #ifdef CONFIG_64BIT if (!test_thread_flag(TIF_32BIT)) - sigframe_size |= 1; + sigframe_size |= 1; /* XXX ???? */ #endif } else { unsigned long psw = USER_PSW; @@ -373,11 +343,11 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, } regs->gr[0] = psw; - regs->iaoq[0] = haddr | 3; + regs->iaoq[0] = haddr | PRIV_USER; regs->iaoq[1] = regs->iaoq[0] + 4; } - regs->gr[2] = rp; /* userland return pointer */ + regs->gr[2] = rp; /* userland return pointer */ regs->gr[26] = ksig->sig; /* signal number */ #ifdef CONFIG_64BIT @@ -391,15 +361,15 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, regs->gr[24] = A(&frame->uc); /* ucontext pointer */ } - DBG(1,"setup_rt_frame: making sigreturn frame: %#lx + %#lx = %#lx\n", + DBG(1, "%s: making sigreturn frame: %#lx + %#lx = %#lx\n", __func__, regs->gr[30], sigframe_size, regs->gr[30] + sigframe_size); /* Raise the user stack pointer to make a proper call frame. */ regs->gr[30] = (A(frame) + sigframe_size); - DBG(1,"setup_rt_frame: sig deliver (%s,%d) frame=0x%p sp=%#lx iaoq=%#lx/%#lx rp=%#lx\n", - current->comm, current->pid, frame, regs->gr[30], + DBG(1, "%s: sig deliver (%s,%d) frame=0x%p sp=%#lx iaoq=%#lx/%#lx rp=%#lx\n", + __func__, current->comm, current->pid, frame, regs->gr[30], regs->iaoq[0], regs->iaoq[1], rp); return 0; @@ -415,8 +385,8 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs, int in_syscall) int ret; sigset_t *oldset = sigmask_to_save(); - DBG(1,"handle_signal: sig=%ld, ka=%p, info=%p, oldset=%p, regs=%p\n", - ksig->sig, ksig->ka, ksig->info, oldset, regs); + DBG(1, "%s: sig=%d, ka=%p, info=%p, oldset=%p, regs=%p\n", + __func__, ksig->sig, &ksig->ka, &ksig->info, oldset, regs); /* Set up the stack frame */ ret = setup_rt_frame(ksig, oldset, regs, in_syscall); @@ -424,8 +394,8 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs, int in_syscall) signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP) || test_thread_flag(TIF_BLOCKSTEP)); - DBG(1,KERN_DEBUG "do_signal: Exit (success), regs->gr[28] = %ld\n", - regs->gr[28]); + DBG(1, "%s: Exit (success), regs->gr[28] = %ld\n", + __func__, regs->gr[28]); } /* @@ -483,21 +453,27 @@ syscall_restart(struct pt_regs *regs, struct k_sigaction *ka) if (regs->orig_r28) return; regs->orig_r28 = 1; /* no more restarts */ + + DBG(1, "%s: orig_r28 = %ld pid %d r20 %ld\n", + __func__, regs->orig_r28, task_pid_nr(current), regs->gr[20]); + /* Check the return code */ switch (regs->gr[28]) { case -ERESTART_RESTARTBLOCK: case -ERESTARTNOHAND: - DBG(1,"ERESTARTNOHAND: returning -EINTR\n"); + DBG(1, "%s: ERESTARTNOHAND: returning -EINTR\n", __func__); regs->gr[28] = -EINTR; break; case -ERESTARTSYS: if (!(ka->sa.sa_flags & SA_RESTART)) { - DBG(1,"ERESTARTSYS: putting -EINTR\n"); + DBG(1, "%s: ERESTARTSYS: putting -EINTR pid %d\n", + __func__, task_pid_nr(current)); regs->gr[28] = -EINTR; break; } fallthrough; case -ERESTARTNOINTR: + DBG(1, "%s: %ld\n", __func__, regs->gr[28]); check_syscallno_in_delay_branch(regs); break; } @@ -509,50 +485,52 @@ insert_restart_trampoline(struct pt_regs *regs) if (regs->orig_r28) return; regs->orig_r28 = 1; /* no more restarts */ - switch(regs->gr[28]) { + + DBG(2, "%s: gr28 = %ld pid %d\n", + __func__, regs->gr[28], task_pid_nr(current)); + + switch (regs->gr[28]) { case -ERESTART_RESTARTBLOCK: { /* Restart the system call - no handlers present */ unsigned int *usp = (unsigned int *)regs->gr[30]; - unsigned long start = (unsigned long) &usp[2]; - unsigned long end = (unsigned long) &usp[5]; + unsigned long rp; long err = 0; /* check that we don't exceed the stack */ - if (A(&usp[0]) >= user_addr_max() - 5 * sizeof(int)) + if (A(&usp[0]) >= TASK_SIZE_MAX - 5 * sizeof(int)) return; - /* Setup a trampoline to restart the syscall - * with __NR_restart_syscall + /* Call trampoline in vdso to restart the syscall + * with __NR_restart_syscall. + * Original return addresses are on stack like this: * * 0: <return address (orig r31)> * 4: <2nd half for 64-bit> - * 8: ldw 0(%sp), %r31 - * 12: be 0x100(%sr2, %r0) - * 16: ldi __NR_restart_syscall, %r20 */ #ifdef CONFIG_64BIT - err |= put_user(regs->gr[31] >> 32, &usp[0]); - err |= put_user(regs->gr[31] & 0xffffffff, &usp[1]); - err |= put_user(0x0fc010df, &usp[2]); -#else - err |= put_user(regs->gr[31], &usp[0]); - err |= put_user(0x0fc0109f, &usp[2]); + if (!is_compat_task()) { + err |= put_user(regs->gr[31] >> 32, &usp[0]); + err |= put_user(regs->gr[31] & 0xffffffff, &usp[1]); + rp = VDSO64_SYMBOL(current, restart_syscall); + } else #endif - err |= put_user(0xe0008200, &usp[3]); - err |= put_user(0x34140000, &usp[4]); - + { + err |= put_user(regs->gr[31], &usp[0]); + rp = VDSO32_SYMBOL(current, restart_syscall); + } WARN_ON(err); - /* flush data/instruction cache for new insns */ - flush_user_dcache_range_asm(start, end); - flush_user_icache_range_asm(start, end); - - regs->gr[31] = regs->gr[30] + 8; + regs->gr[31] = rp; + DBG(1, "%s: ERESTART_RESTARTBLOCK\n", __func__); return; } + case -EINTR: + /* ok, was handled before and should be returned. */ + break; case -ERESTARTNOHAND: case -ERESTARTSYS: case -ERESTARTNOINTR: + DBG(1, "%s: Type %ld\n", __func__, regs->gr[28]); check_syscallno_in_delay_branch(regs); return; default: @@ -567,30 +545,35 @@ insert_restart_trampoline(struct pt_regs *regs) * registers). As noted below, the syscall number gets restored for * us due to the magic of delayed branching. */ -asmlinkage void -do_signal(struct pt_regs *regs, long in_syscall) +static void do_signal(struct pt_regs *regs, long in_syscall) { struct ksignal ksig; + int restart_syscall; + bool has_handler; - DBG(1,"\ndo_signal: regs=0x%p, sr7 %#lx, in_syscall=%d\n", - regs, regs->sr[7], in_syscall); + has_handler = get_signal(&ksig); - if (get_signal(&ksig)) { - DBG(3,"do_signal: signr = %d, regs->gr[28] = %ld\n", signr, regs->gr[28]); + restart_syscall = 0; + if (in_syscall) + restart_syscall = 1; + + if (has_handler) { /* Restart a system call if necessary. */ - if (in_syscall) + if (restart_syscall) syscall_restart(regs, &ksig.ka); handle_signal(&ksig, regs, in_syscall); + DBG(1, "%s: Handled signal pid %d\n", + __func__, task_pid_nr(current)); return; } - /* Did we come from a system call? */ - if (in_syscall) + /* Do we need to restart the system call? */ + if (restart_syscall) insert_restart_trampoline(regs); - DBG(1,"do_signal: Exit (not delivered), regs->gr[28] = %ld\n", - regs->gr[28]); + DBG(1, "%s: Exit (not delivered), regs->gr[28] = %ld orig_r28 = %ld pid %d\n", + __func__, regs->gr[28], regs->orig_r28, task_pid_nr(current)); restore_saved_sigmask(); } diff --git a/arch/parisc/kernel/signal32.h b/arch/parisc/kernel/signal32.h index f166250f2d06..c03eb1ed4c53 100644 --- a/arch/parisc/kernel/signal32.h +++ b/arch/parisc/kernel/signal32.h @@ -36,21 +36,12 @@ struct compat_regfile { compat_int_t rf_sar; }; -#define COMPAT_SIGRETURN_TRAMP 4 -#define COMPAT_SIGRESTARTBLOCK_TRAMP 5 -#define COMPAT_TRAMP_SIZE (COMPAT_SIGRETURN_TRAMP + \ - COMPAT_SIGRESTARTBLOCK_TRAMP) - struct compat_rt_sigframe { - /* XXX: Must match trampoline size in arch/parisc/kernel/signal.c - Secondary to that it must protect the ERESTART_RESTARTBLOCK - trampoline we left on the stack (we were bad and didn't - change sp so we could run really fast.) */ - compat_uint_t tramp[COMPAT_TRAMP_SIZE]; - compat_siginfo_t info; - struct compat_ucontext uc; - /* Hidden location of truncated registers, *must* be last. */ - struct compat_regfile regs; + unsigned int tramp[2]; /* holds original return address */ + compat_siginfo_t info; + struct compat_ucontext uc; + /* Hidden location of truncated registers, *must* be last. */ + struct compat_regfile regs; }; /* diff --git a/arch/parisc/kernel/topology.c b/arch/parisc/kernel/topology.c index 0a10e4ddc528..e88a6ce7c96d 100644 --- a/arch/parisc/kernel/topology.c +++ b/arch/parisc/kernel/topology.c @@ -101,8 +101,8 @@ void __init store_cpu_topology(unsigned int cpuid) update_siblings_masks(cpuid); - pr_info("CPU%u: thread %d, cpu %d, socket %d\n", - cpuid, cpu_topology[cpuid].thread_id, + pr_info("CPU%u: cpu core %d of socket %d\n", + cpuid, cpu_topology[cpuid].core_id, cpu_topology[cpuid].socket_id); } diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index b6fdebddc8e9..a6e61cf2cad0 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -302,7 +302,10 @@ static void handle_break(struct pt_regs *regs) parisc_kprobe_break_handler(regs); return; } - + if (unlikely(iir == PARISC_KPROBES_BREAK_INSN2)) { + parisc_kprobe_ss_handler(regs); + return; + } #endif #ifdef CONFIG_KGDB @@ -539,11 +542,6 @@ void notrace handle_interruption(int code, struct pt_regs *regs) /* Recovery counter trap */ regs->gr[0] &= ~PSW_R; -#ifdef CONFIG_KPROBES - if (parisc_kprobe_ss_handler(regs)) - return; -#endif - #ifdef CONFIG_KGDB if (kgdb_single_step) { kgdb_handle_exception(0, SIGTRAP, 0, regs); @@ -662,6 +660,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs) by hand. Technically we need to emulate: fdc,fdce,pdc,"fic,4f",prober,probeir,probew, probeiw */ + if (code == 17 && handle_nadtlb_fault(regs)) + return; fault_address = regs->ior; fault_space = regs->isr; break; diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c index 237d20dd5622..ed1e88a74dc4 100644 --- a/arch/parisc/kernel/unaligned.c +++ b/arch/parisc/kernel/unaligned.c @@ -3,14 +3,11 @@ * Unaligned memory access handler * * Copyright (C) 2001 Randolph Chung <tausq@debian.org> + * Copyright (C) 2022 Helge Deller <deller@gmx.de> * Significantly tweaked by LaMont Jones <lamont@debian.org> */ -#include <linux/jiffies.h> -#include <linux/kernel.h> -#include <linux/module.h> #include <linux/sched/signal.h> -#include <linux/sched/debug.h> #include <linux/signal.h> #include <linux/ratelimit.h> #include <linux/uaccess.h> @@ -25,18 +22,7 @@ #define DPRINTF(fmt, args...) #endif -#ifdef CONFIG_64BIT -#define RFMT "%016lx" -#else -#define RFMT "%08lx" -#endif - -#define FIXUP_BRANCH(lbl) \ - "\tldil L%%" #lbl ", %%r1\n" \ - "\tldo R%%" #lbl "(%%r1), %%r1\n" \ - "\tbv,n %%r0(%%r1)\n" -/* If you use FIXUP_BRANCH, then you must list this clobber */ -#define FIXUP_BRANCH_CLOBBER "r1" +#define RFMT "%#08lx" /* 1111 1100 0000 0000 0001 0011 1100 0000 */ #define OPCODE1(a,b,c) ((a)<<26|(b)<<12|(c)<<6) @@ -114,37 +100,30 @@ #define IM14(i) IM((i),14) #define ERR_NOTHANDLED -1 -#define ERR_PAGEFAULT -2 int unaligned_enabled __read_mostly = 1; static int emulate_ldh(struct pt_regs *regs, int toreg) { unsigned long saddr = regs->ior; - unsigned long val = 0; - int ret; + unsigned long val = 0, temp1; + ASM_EXCEPTIONTABLE_VAR(ret); DPRINTF("load " RFMT ":" RFMT " to r%d for 2 bytes\n", regs->isr, regs->ior, toreg); __asm__ __volatile__ ( " mtsp %4, %%sr1\n" -"1: ldbs 0(%%sr1,%3), %%r20\n" +"1: ldbs 0(%%sr1,%3), %2\n" "2: ldbs 1(%%sr1,%3), %0\n" -" depw %%r20, 23, 24, %0\n" -" copy %%r0, %1\n" +" depw %2, 23, 24, %0\n" "3: \n" -" .section .fixup,\"ax\"\n" -"4: ldi -2, %1\n" - FIXUP_BRANCH(3b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b, 4b) - ASM_EXCEPTIONTABLE_ENTRY(2b, 4b) - : "=r" (val), "=r" (ret) - : "0" (val), "r" (saddr), "r" (regs->isr) - : "r20", FIXUP_BRANCH_CLOBBER ); + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 3b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 3b) + : "+r" (val), "+r" (ret), "=&r" (temp1) + : "r" (saddr), "r" (regs->isr) ); - DPRINTF("val = 0x" RFMT "\n", val); + DPRINTF("val = " RFMT "\n", val); if (toreg) regs->gr[toreg] = val; @@ -155,34 +134,28 @@ static int emulate_ldh(struct pt_regs *regs, int toreg) static int emulate_ldw(struct pt_regs *regs, int toreg, int flop) { unsigned long saddr = regs->ior; - unsigned long val = 0; - int ret; + unsigned long val = 0, temp1, temp2; + ASM_EXCEPTIONTABLE_VAR(ret); DPRINTF("load " RFMT ":" RFMT " to r%d for 4 bytes\n", regs->isr, regs->ior, toreg); __asm__ __volatile__ ( -" zdep %3,28,2,%%r19\n" /* r19=(ofs&3)*8 */ -" mtsp %4, %%sr1\n" -" depw %%r0,31,2,%3\n" -"1: ldw 0(%%sr1,%3),%0\n" -"2: ldw 4(%%sr1,%3),%%r20\n" -" subi 32,%%r19,%%r19\n" -" mtctl %%r19,11\n" -" vshd %0,%%r20,%0\n" -" copy %%r0, %1\n" +" zdep %4,28,2,%2\n" /* r19=(ofs&3)*8 */ +" mtsp %5, %%sr1\n" +" depw %%r0,31,2,%4\n" +"1: ldw 0(%%sr1,%4),%0\n" +"2: ldw 4(%%sr1,%4),%3\n" +" subi 32,%4,%2\n" +" mtctl %2,11\n" +" vshd %0,%3,%0\n" "3: \n" -" .section .fixup,\"ax\"\n" -"4: ldi -2, %1\n" - FIXUP_BRANCH(3b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b, 4b) - ASM_EXCEPTIONTABLE_ENTRY(2b, 4b) - : "=r" (val), "=r" (ret) - : "0" (val), "r" (saddr), "r" (regs->isr) - : "r19", "r20", FIXUP_BRANCH_CLOBBER ); + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 3b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 3b) + : "+r" (val), "+r" (ret), "=&r" (temp1), "=&r" (temp2) + : "r" (saddr), "r" (regs->isr) ); - DPRINTF("val = 0x" RFMT "\n", val); + DPRINTF("val = " RFMT "\n", val); if (flop) ((__u32*)(regs->fr))[toreg] = val; @@ -195,16 +168,15 @@ static int emulate_ldd(struct pt_regs *regs, int toreg, int flop) { unsigned long saddr = regs->ior; __u64 val = 0; - int ret; + ASM_EXCEPTIONTABLE_VAR(ret); DPRINTF("load " RFMT ":" RFMT " to r%d for 8 bytes\n", regs->isr, regs->ior, toreg); -#ifdef CONFIG_PA20 -#ifndef CONFIG_64BIT - if (!flop) - return -1; -#endif + if (!IS_ENABLED(CONFIG_64BIT) && !flop) + return ERR_NOTHANDLED; + +#ifdef CONFIG_64BIT __asm__ __volatile__ ( " depd,z %3,60,3,%%r19\n" /* r19=(ofs&7)*8 */ " mtsp %4, %%sr1\n" @@ -214,44 +186,32 @@ static int emulate_ldd(struct pt_regs *regs, int toreg, int flop) " subi 64,%%r19,%%r19\n" " mtsar %%r19\n" " shrpd %0,%%r20,%%sar,%0\n" -" copy %%r0, %1\n" "3: \n" -" .section .fixup,\"ax\"\n" -"4: ldi -2, %1\n" - FIXUP_BRANCH(3b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,4b) - ASM_EXCEPTIONTABLE_ENTRY(2b,4b) - : "=r" (val), "=r" (ret) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 3b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 3b) + : "=r" (val), "+r" (ret) : "0" (val), "r" (saddr), "r" (regs->isr) - : "r19", "r20", FIXUP_BRANCH_CLOBBER ); + : "r19", "r20" ); #else { - unsigned long valh=0,vall=0; + unsigned long shift, temp1; __asm__ __volatile__ ( -" zdep %5,29,2,%%r19\n" /* r19=(ofs&3)*8 */ -" mtsp %6, %%sr1\n" -" dep %%r0,31,2,%5\n" -"1: ldw 0(%%sr1,%5),%0\n" -"2: ldw 4(%%sr1,%5),%1\n" -"3: ldw 8(%%sr1,%5),%%r20\n" -" subi 32,%%r19,%%r19\n" -" mtsar %%r19\n" -" vshd %0,%1,%0\n" -" vshd %1,%%r20,%1\n" -" copy %%r0, %2\n" +" zdep %2,29,2,%3\n" /* r19=(ofs&3)*8 */ +" mtsp %5, %%sr1\n" +" dep %%r0,31,2,%2\n" +"1: ldw 0(%%sr1,%2),%0\n" +"2: ldw 4(%%sr1,%2),%R0\n" +"3: ldw 8(%%sr1,%2),%4\n" +" subi 32,%3,%3\n" +" mtsar %3\n" +" vshd %0,%R0,%0\n" +" vshd %R0,%4,%R0\n" "4: \n" -" .section .fixup,\"ax\"\n" -"5: ldi -2, %2\n" - FIXUP_BRANCH(4b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,5b) - ASM_EXCEPTIONTABLE_ENTRY(2b,5b) - ASM_EXCEPTIONTABLE_ENTRY(3b,5b) - : "=r" (valh), "=r" (vall), "=r" (ret) - : "0" (valh), "1" (vall), "r" (saddr), "r" (regs->isr) - : "r19", "r20", FIXUP_BRANCH_CLOBBER ); - val=((__u64)valh<<32)|(__u64)vall; + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 4b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 4b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(3b, 4b) + : "+r" (val), "+r" (ret), "+r" (saddr), "=&r" (shift), "=&r" (temp1) + : "r" (regs->isr) ); } #endif @@ -267,31 +227,25 @@ static int emulate_ldd(struct pt_regs *regs, int toreg, int flop) static int emulate_sth(struct pt_regs *regs, int frreg) { - unsigned long val = regs->gr[frreg]; - int ret; + unsigned long val = regs->gr[frreg], temp1; + ASM_EXCEPTIONTABLE_VAR(ret); if (!frreg) val = 0; - DPRINTF("store r%d (0x" RFMT ") to " RFMT ":" RFMT " for 2 bytes\n", frreg, + DPRINTF("store r%d (" RFMT ") to " RFMT ":" RFMT " for 2 bytes\n", frreg, val, regs->isr, regs->ior); __asm__ __volatile__ ( -" mtsp %3, %%sr1\n" -" extrw,u %1, 23, 8, %%r19\n" -"1: stb %1, 1(%%sr1, %2)\n" -"2: stb %%r19, 0(%%sr1, %2)\n" -" copy %%r0, %0\n" +" mtsp %4, %%sr1\n" +" extrw,u %2, 23, 8, %1\n" +"1: stb %1, 0(%%sr1, %3)\n" +"2: stb %2, 1(%%sr1, %3)\n" "3: \n" -" .section .fixup,\"ax\"\n" -"4: ldi -2, %0\n" - FIXUP_BRANCH(3b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,4b) - ASM_EXCEPTIONTABLE_ENTRY(2b,4b) - : "=r" (ret) - : "r" (val), "r" (regs->ior), "r" (regs->isr) - : "r19", FIXUP_BRANCH_CLOBBER ); + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 3b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 3b) + : "+r" (ret), "=&r" (temp1) + : "r" (val), "r" (regs->ior), "r" (regs->isr) ); return ret; } @@ -299,7 +253,7 @@ static int emulate_sth(struct pt_regs *regs, int frreg) static int emulate_stw(struct pt_regs *regs, int frreg, int flop) { unsigned long val; - int ret; + ASM_EXCEPTIONTABLE_VAR(ret); if (flop) val = ((__u32*)(regs->fr))[frreg]; @@ -308,7 +262,7 @@ static int emulate_stw(struct pt_regs *regs, int frreg, int flop) else val = 0; - DPRINTF("store r%d (0x" RFMT ") to " RFMT ":" RFMT " for 4 bytes\n", frreg, + DPRINTF("store r%d (" RFMT ") to " RFMT ":" RFMT " for 4 bytes\n", frreg, val, regs->isr, regs->ior); @@ -328,24 +282,19 @@ static int emulate_stw(struct pt_regs *regs, int frreg, int flop) " or %%r1, %%r21, %%r21\n" " stw %%r20,0(%%sr1,%2)\n" " stw %%r21,4(%%sr1,%2)\n" -" copy %%r0, %0\n" "3: \n" -" .section .fixup,\"ax\"\n" -"4: ldi -2, %0\n" - FIXUP_BRANCH(3b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,4b) - ASM_EXCEPTIONTABLE_ENTRY(2b,4b) - : "=r" (ret) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 3b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 3b) + : "+r" (ret) : "r" (val), "r" (regs->ior), "r" (regs->isr) - : "r19", "r20", "r21", "r22", "r1", FIXUP_BRANCH_CLOBBER ); + : "r19", "r20", "r21", "r22", "r1" ); - return 0; + return ret; } static int emulate_std(struct pt_regs *regs, int frreg, int flop) { __u64 val; - int ret; + ASM_EXCEPTIONTABLE_VAR(ret); if (flop) val = regs->fr[frreg]; @@ -357,11 +306,10 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop) DPRINTF("store r%d (0x%016llx) to " RFMT ":" RFMT " for 8 bytes\n", frreg, val, regs->isr, regs->ior); -#ifdef CONFIG_PA20 -#ifndef CONFIG_64BIT - if (!flop) - return -1; -#endif + if (!IS_ENABLED(CONFIG_64BIT) && !flop) + return ERR_NOTHANDLED; + +#ifdef CONFIG_64BIT __asm__ __volatile__ ( " mtsp %3, %%sr1\n" " depd,z %2, 60, 3, %%r19\n" @@ -378,26 +326,21 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop) " or %%r1, %%r21, %%r21\n" "3: std %%r20,0(%%sr1,%2)\n" "4: std %%r21,8(%%sr1,%2)\n" -" copy %%r0, %0\n" "5: \n" -" .section .fixup,\"ax\"\n" -"6: ldi -2, %0\n" - FIXUP_BRANCH(5b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,6b) - ASM_EXCEPTIONTABLE_ENTRY(2b,6b) - ASM_EXCEPTIONTABLE_ENTRY(3b,6b) - ASM_EXCEPTIONTABLE_ENTRY(4b,6b) - : "=r" (ret) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 5b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 5b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(3b, 5b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(4b, 5b) + : "+r" (ret) : "r" (val), "r" (regs->ior), "r" (regs->isr) - : "r19", "r20", "r21", "r22", "r1", FIXUP_BRANCH_CLOBBER ); + : "r19", "r20", "r21", "r22", "r1" ); #else { unsigned long valh=(val>>32),vall=(val&0xffffffffl); __asm__ __volatile__ ( " mtsp %4, %%sr1\n" " zdep %2, 29, 2, %%r19\n" -" dep %%r0, 31, 2, %2\n" +" dep %%r0, 31, 2, %3\n" " mtsar %%r19\n" " zvdepi -2, 32, %%r19\n" "1: ldw 0(%%sr1,%3),%%r20\n" @@ -409,23 +352,18 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop) " andcm %%r21, %%r19, %%r21\n" " or %1, %%r20, %1\n" " or %2, %%r21, %2\n" -"3: stw %1,0(%%sr1,%1)\n" +"3: stw %1,0(%%sr1,%3)\n" "4: stw %%r1,4(%%sr1,%3)\n" "5: stw %2,8(%%sr1,%3)\n" -" copy %%r0, %0\n" "6: \n" -" .section .fixup,\"ax\"\n" -"7: ldi -2, %0\n" - FIXUP_BRANCH(6b) -" .previous\n" - ASM_EXCEPTIONTABLE_ENTRY(1b,7b) - ASM_EXCEPTIONTABLE_ENTRY(2b,7b) - ASM_EXCEPTIONTABLE_ENTRY(3b,7b) - ASM_EXCEPTIONTABLE_ENTRY(4b,7b) - ASM_EXCEPTIONTABLE_ENTRY(5b,7b) - : "=r" (ret) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 6b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 6b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(3b, 6b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(4b, 6b) + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(5b, 6b) + : "+r" (ret) : "r" (valh), "r" (vall), "r" (regs->ior), "r" (regs->isr) - : "r19", "r20", "r21", "r1", FIXUP_BRANCH_CLOBBER ); + : "r19", "r20", "r21", "r1" ); } #endif @@ -438,7 +376,6 @@ void handle_unaligned(struct pt_regs *regs) unsigned long newbase = R1(regs->iir)?regs->gr[R1(regs->iir)]:0; int modify = 0; int ret = ERR_NOTHANDLED; - register int flop=0; /* true if this is a flop */ __inc_irq_stat(irq_unaligned_count); @@ -450,10 +387,10 @@ void handle_unaligned(struct pt_regs *regs) if (!(current->thread.flags & PARISC_UAC_NOPRINT) && __ratelimit(&ratelimit)) { - char buf[256]; - sprintf(buf, "%s(%d): unaligned access to 0x" RFMT " at ip=0x" RFMT "\n", - current->comm, task_pid_nr(current), regs->ior, regs->iaoq[0]); - printk(KERN_WARNING "%s", buf); + printk(KERN_WARNING "%s(%d): unaligned access to " RFMT + " at ip " RFMT " (iir " RFMT ")\n", + current->comm, task_pid_nr(current), regs->ior, + regs->iaoq[0], regs->iir); #ifdef DEBUG_UNALIGNED show_regs(regs); #endif @@ -547,7 +484,7 @@ void handle_unaligned(struct pt_regs *regs) ret = emulate_stw(regs, R2(regs->iir),0); break; -#ifdef CONFIG_PA20 +#ifdef CONFIG_64BIT case OPCODE_LDD_I: case OPCODE_LDDA_I: case OPCODE_LDD_S: @@ -565,13 +502,11 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_FLDWS: case OPCODE_FLDWXR: case OPCODE_FLDWSR: - flop=1; ret = emulate_ldw(regs,FR3(regs->iir),1); break; case OPCODE_FLDDX: case OPCODE_FLDDS: - flop=1; ret = emulate_ldd(regs,R3(regs->iir),1); break; @@ -579,13 +514,11 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_FSTWS: case OPCODE_FSTWXR: case OPCODE_FSTWSR: - flop=1; ret = emulate_stw(regs,FR3(regs->iir),1); break; case OPCODE_FSTDX: case OPCODE_FSTDS: - flop=1; ret = emulate_std(regs,R3(regs->iir),1); break; @@ -596,37 +529,33 @@ void handle_unaligned(struct pt_regs *regs) ret = ERR_NOTHANDLED; /* "undefined", but lets kill them. */ break; } -#ifdef CONFIG_PA20 switch (regs->iir & OPCODE2_MASK) { case OPCODE_FLDD_L: - flop=1; ret = emulate_ldd(regs,R2(regs->iir),1); break; case OPCODE_FSTD_L: - flop=1; ret = emulate_std(regs, R2(regs->iir),1); break; +#ifdef CONFIG_64BIT case OPCODE_LDD_L: ret = emulate_ldd(regs, R2(regs->iir),0); break; case OPCODE_STD_L: ret = emulate_std(regs, R2(regs->iir),0); break; - } #endif + } switch (regs->iir & OPCODE3_MASK) { case OPCODE_FLDW_L: - flop=1; - ret = emulate_ldw(regs, R2(regs->iir),0); + ret = emulate_ldw(regs, R2(regs->iir), 1); break; case OPCODE_LDW_M: - ret = emulate_ldw(regs, R2(regs->iir),1); + ret = emulate_ldw(regs, R2(regs->iir), 0); break; case OPCODE_FSTW_L: - flop=1; ret = emulate_stw(regs, R2(regs->iir),1); break; case OPCODE_STW_M: @@ -673,7 +602,7 @@ void handle_unaligned(struct pt_regs *regs) printk(KERN_CRIT "Unaligned handler failed, ret = %d\n", ret); die_if_kernel("Unaligned data reference", regs, 28); - if (ret == ERR_PAGEFAULT) + if (ret == -EFAULT) { force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->ior); diff --git a/arch/parisc/kernel/vdso.c b/arch/parisc/kernel/vdso.c new file mode 100644 index 000000000000..63dc44c4c246 --- /dev/null +++ b/arch/parisc/kernel/vdso.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022 Helge Deller <deller@gmx.de> + * + * based on arch/s390/kernel/vdso.c which is + * Copyright IBM Corp. 2008 + * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/elf.h> +#include <linux/timekeeper_internal.h> +#include <linux/compat.h> +#include <linux/nsproxy.h> +#include <linux/time_namespace.h> +#include <linux/random.h> + +#include <asm/pgtable.h> +#include <asm/page.h> +#include <asm/sections.h> +#include <asm/vdso.h> +#include <asm/cacheflush.h> + +extern char vdso32_start, vdso32_end; +extern char vdso64_start, vdso64_end; + +static int vdso_mremap(const struct vm_special_mapping *sm, + struct vm_area_struct *vma) +{ + current->mm->context.vdso_base = vma->vm_start; + return 0; +} + +#ifdef CONFIG_64BIT +static struct vm_special_mapping vdso64_mapping = { + .name = "[vdso]", + .mremap = vdso_mremap, +}; +#endif + +static struct vm_special_mapping vdso32_mapping = { + .name = "[vdso]", + .mremap = vdso_mremap, +}; + +/* + * This is called from binfmt_elf, we create the special vma for the + * vDSO and insert it into the mm struct tree + */ +int arch_setup_additional_pages(struct linux_binprm *bprm, + int executable_stack) +{ + + unsigned long vdso_text_start, vdso_text_len, map_base; + struct vm_special_mapping *vdso_mapping; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + int rc; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + +#ifdef CONFIG_64BIT + if (!is_compat_task()) { + vdso_text_len = &vdso64_end - &vdso64_start; + vdso_mapping = &vdso64_mapping; + } else +#endif + { + vdso_text_len = &vdso32_end - &vdso32_start; + vdso_mapping = &vdso32_mapping; + } + + map_base = mm->mmap_base; + if (current->flags & PF_RANDOMIZE) + map_base -= (get_random_int() & 0x1f) * PAGE_SIZE; + + vdso_text_start = get_unmapped_area(NULL, map_base, vdso_text_len, 0, 0); + + /* VM_MAYWRITE for COW so gdb can set breakpoints */ + vma = _install_special_mapping(mm, vdso_text_start, vdso_text_len, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + vdso_mapping); + if (IS_ERR(vma)) { + do_munmap(mm, vdso_text_start, PAGE_SIZE, NULL); + rc = PTR_ERR(vma); + } else { + current->mm->context.vdso_base = vdso_text_start; + rc = 0; + } + + mmap_write_unlock(mm); + return rc; +} + +static struct page ** __init vdso_setup_pages(void *start, void *end) +{ + int pages = (end - start) >> PAGE_SHIFT; + struct page **pagelist; + int i; + + pagelist = kcalloc(pages + 1, sizeof(struct page *), GFP_KERNEL); + if (!pagelist) + panic("%s: Cannot allocate page list for VDSO", __func__); + for (i = 0; i < pages; i++) + pagelist[i] = virt_to_page(start + i * PAGE_SIZE); + return pagelist; +} + +static int __init vdso_init(void) +{ +#ifdef CONFIG_64BIT + vdso64_mapping.pages = vdso_setup_pages(&vdso64_start, &vdso64_end); +#endif + if (IS_ENABLED(CONFIG_COMPAT) || !IS_ENABLED(CONFIG_64BIT)) + vdso32_mapping.pages = vdso_setup_pages(&vdso32_start, &vdso32_end); + return 0; +} +arch_initcall(vdso_init); diff --git a/arch/parisc/kernel/vdso32/Makefile b/arch/parisc/kernel/vdso32/Makefile new file mode 100644 index 000000000000..85b1c6d261d1 --- /dev/null +++ b/arch/parisc/kernel/vdso32/Makefile @@ -0,0 +1,53 @@ +# List of files in the vdso, has to be asm only for now + +obj-vdso32 = note.o sigtramp.o restart_syscall.o + +# Build rules + +targets := $(obj-vdso32) vdso32.so +obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32)) + +ccflags-y := -shared -fno-common -fbuiltin -mno-fast-indirect-calls -O2 -mno-long-calls +# -march=1.1 -mschedule=7100LC +ccflags-y += -nostdlib -Wl,-soname=linux-vdso32.so.1 \ + $(call ld-option, -Wl$(comma)--hash-style=sysv) +asflags-y := -D__VDSO32__ -s + +KBUILD_AFLAGS += -DBUILD_VDSO +KBUILD_CFLAGS += -DBUILD_VDSO -DDISABLE_BRANCH_PROFILING + +VDSO_LIBGCC := $(shell $(CROSS32CC) -print-libgcc-file-name) + +obj-y += vdso32_wrapper.o +extra-y += vdso32.lds +CPPFLAGS_vdso32.lds += -P -C # -U$(ARCH) + +$(obj)/vdso32_wrapper.o : $(obj)/vdso32.so FORCE + +# Force dependency (incbin is bad) +# link rule for the .so file, .lds has to be first +$(obj)/vdso32.so: $(src)/vdso32.lds $(obj-vdso32) $(obj-cvdso32) $(VDSO_LIBGCC) + $(call if_changed,vdso32ld) + +# assembly rules for the .S files +$(obj-vdso32): %.o: %.S FORCE + $(call if_changed_dep,vdso32as) + +$(obj-cvdso32): %.o: %.c FORCE + $(call if_changed_dep,vdso32cc) + +# actual build commands +quiet_cmd_vdso32ld = VDSO32L $@ + cmd_vdso32ld = $(CROSS32CC) $(c_flags) -Wl,-T $^ -o $@ +quiet_cmd_vdso32as = VDSO32A $@ + cmd_vdso32as = $(CROSS32CC) $(a_flags) -c -o $@ $< +quiet_cmd_vdso32cc = VDSO32C $@ + cmd_vdso32cc = $(CROSS32CC) $(c_flags) -c -fPIC -mno-fast-indirect-calls -o $@ $< + +# Generate VDSO offsets using helper script +gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +quiet_cmd_vdsosym = VDSOSYM $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + +include/generated/vdso32-offsets.h: $(obj)/vdso32.so FORCE + $(call if_changed,vdsosym) diff --git a/arch/nds32/kernel/vdso/gen_vdso_offsets.sh b/arch/parisc/kernel/vdso32/gen_vdso_offsets.sh index 01924ff071ad..da39d6cff7f1 100755 --- a/arch/nds32/kernel/vdso/gen_vdso_offsets.sh +++ b/arch/parisc/kernel/vdso32/gen_vdso_offsets.sh @@ -1,4 +1,5 @@ #!/bin/sh +# SPDX-License-Identifier: GPL-2.0 # # Match symbols in the DSO that look like VDSO_*; produce a header file @@ -7,9 +8,8 @@ # Doing this inside the Makefile will break the $(filter-out) function, # causing Kbuild to rebuild the vdso-offsets header file every time. # -# Author: Will Deacon <will.deacon@arm.com +# Inspired by arm64 version. # LC_ALL=C -sed -n -e 's/^00*/0/' -e \ -'s/^\([0-9a-fA-F]*\) . VDSO_\([a-zA-Z0-9_]*\)$/\#define vdso_offset_\2\t0x\1/p' +sed -n 's/\([0-9a-f]*\) . __kernel_\(.*\)/\#define vdso32_offset_\2\t0x\1/p' diff --git a/arch/parisc/kernel/vdso32/note.S b/arch/parisc/kernel/vdso32/note.S new file mode 100644 index 000000000000..bb350918bebd --- /dev/null +++ b/arch/parisc/kernel/vdso32/note.S @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include <linux/uts.h> +#include <linux/version.h> + +#define ASM_ELF_NOTE_BEGIN(name, flags, vendor, type) \ + .section name, flags; \ + .balign 4; \ + .long 1f - 0f; /* name length */ \ + .long 3f - 2f; /* data length */ \ + .long type; /* note type */ \ +0: .asciz vendor; /* vendor name */ \ +1: .balign 4; \ +2: + +#define ASM_ELF_NOTE_END \ +3: .balign 4; /* pad out section */ \ + .previous + + ASM_ELF_NOTE_BEGIN(".note.kernel-version", "a", UTS_SYSNAME, 0) + .long LINUX_VERSION_CODE + ASM_ELF_NOTE_END diff --git a/arch/parisc/kernel/vdso32/restart_syscall.S b/arch/parisc/kernel/vdso32/restart_syscall.S new file mode 100644 index 000000000000..7e82008d7e40 --- /dev/null +++ b/arch/parisc/kernel/vdso32/restart_syscall.S @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Syscall restart trampoline for 32 and 64 bits processes. + * + * Copyright (C) 2018-2022 Helge Deller <deller@gmx.de> + * Copyright (C) 2022 John David Anglin <dave.anglin@bell.net> + */ + +#include <asm/unistd.h> +#include <asm/vdso.h> + +#include <linux/linkage.h> + + .text + +ENTRY_CFI(__kernel_restart_syscall) + /* + * Setup a trampoline to restart the syscall + * with __NR_restart_syscall + */ + + /* load return pointer */ +#if defined(__VDSO64__) + ldd 0(%sp), %r31 +#elif defined(__VDSO32__) + ldw 0(%sp), %r31 +#endif + + be 0x100(%sr2, %r0) + ldi __NR_restart_syscall, %r20 + +ENDPROC_CFI(__kernel_restart_syscall) diff --git a/arch/parisc/kernel/vdso32/sigtramp.S b/arch/parisc/kernel/vdso32/sigtramp.S new file mode 100644 index 000000000000..192da7077869 --- /dev/null +++ b/arch/parisc/kernel/vdso32/sigtramp.S @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Signal trampolines for 32 bit processes. + * + * Copyright (C) 2006 Randolph Chung <tausq@debian.org> + * Copyright (C) 2018-2022 Helge Deller <deller@gmx.de> + * Copyright (C) 2022 John David Anglin <dave.anglin@bell.net> + */ +#include <asm/unistd.h> +#include <linux/linkage.h> +#include <generated/asm-offsets.h> + + .text + +/* Gdb expects the trampoline is on the stack and the pc is offset from + a 64-byte boundary by 0, 4 or 5 instructions. Since the vdso trampoline + is not on the stack, we need a new variant with different offsets and + data to tell gdb where to find the signal context on the stack. + + Here we put the offset to the context data at the start of the trampoline + region and offset the first trampoline by 2 instructions. Please do + not change the trampoline as the code in gdb depends on the following + instruction sequence exactly. + */ + .align 64 + .word SIGFRAME_CONTEXT_REGS32 + +/* The nop here is a hack. The dwarf2 unwind routines subtract 1 from + the return address to get an address in the middle of the presumed + call instruction. Since we don't have a call here, we artifically + extend the range covered by the unwind info by adding a nop before + the real start. + */ + nop + + .globl __kernel_sigtramp_rt + .type __kernel_sigtramp_rt, @function +__kernel_sigtramp_rt: + .proc + .callinfo FRAME=ASM_SIGFRAME_SIZE32,CALLS,SAVE_RP + .entry + +.Lsigrt_start = . - 4 +0: ldi 0, %r25 /* (in_syscall=0) */ + ldi __NR_rt_sigreturn, %r20 + ble 0x100(%sr2, %r0) + nop + +1: ldi 1, %r25 /* (in_syscall=1) */ + ldi __NR_rt_sigreturn, %r20 + ble 0x100(%sr2, %r0) + nop +.Lsigrt_end: + .exit + .procend + .size __kernel_sigtramp_rt,.-__kernel_sigtramp_rt + + + .section .eh_frame,"a",@progbits + +/* This is where the mcontext_t struct can be found on the stack. */ +#define PTREGS SIGFRAME_CONTEXT_REGS32 /* 32-bit process offset is -672 */ + +/* Register REGNO can be found at offset OFS of the mcontext_t structure. */ + .macro rsave regno,ofs + .byte 0x05 /* DW_CFA_offset_extended */ + .uleb128 \regno; /* regno */ + .uleb128 \ofs /* factored offset */ + .endm + +.Lcie: + .long .Lcie_end - .Lcie_start +.Lcie_start: + .long 0 /* CIE ID */ + .byte 1 /* Version number */ + .stringz "zRS" /* NUL-terminated augmentation string */ + .uleb128 4 /* Code alignment factor */ + .sleb128 4 /* Data alignment factor */ + .byte 89 /* Return address register column, iaoq[0] */ + .uleb128 1 /* Augmentation value length */ + .byte 0x1b /* DW_EH_PE_pcrel | DW_EH_PE_sdata4. */ + .byte 0x0f /* DW_CFA_def_cfa_expresion */ + .uleb128 9f - 1f /* length */ +1: + .byte 0x8e /* DW_OP_breg30 */ + .sleb128 PTREGS +9: + .balign 4 +.Lcie_end: + + .long .Lfde0_end - .Lfde0_start +.Lfde0_start: + .long .Lfde0_start - .Lcie /* CIE pointer. */ + .long .Lsigrt_start - . /* PC start, length */ + .long .Lsigrt_end - .Lsigrt_start + .uleb128 0 /* Augmentation */ + + /* General registers */ + rsave 1, 2 + rsave 2, 3 + rsave 3, 4 + rsave 4, 5 + rsave 5, 6 + rsave 6, 7 + rsave 7, 8 + rsave 8, 9 + rsave 9, 10 + rsave 10, 11 + rsave 11, 12 + rsave 12, 13 + rsave 13, 14 + rsave 14, 15 + rsave 15, 16 + rsave 16, 17 + rsave 17, 18 + rsave 18, 19 + rsave 19, 20 + rsave 20, 21 + rsave 21, 22 + rsave 22, 23 + rsave 23, 24 + rsave 24, 25 + rsave 25, 26 + rsave 26, 27 + rsave 27, 28 + rsave 28, 29 + rsave 29, 30 + rsave 30, 31 + rsave 31, 32 + + /* Floating-point registers */ + rsave 32, 42 + rsave 33, 43 + rsave 34, 44 + rsave 35, 45 + rsave 36, 46 + rsave 37, 47 + rsave 38, 48 + rsave 39, 49 + rsave 40, 50 + rsave 41, 51 + rsave 42, 52 + rsave 43, 53 + rsave 44, 54 + rsave 45, 55 + rsave 46, 56 + rsave 47, 57 + rsave 48, 58 + rsave 49, 59 + rsave 50, 60 + rsave 51, 61 + rsave 52, 62 + rsave 53, 63 + rsave 54, 64 + rsave 55, 65 + rsave 56, 66 + rsave 57, 67 + rsave 58, 68 + rsave 59, 69 + rsave 60, 70 + rsave 61, 71 + rsave 62, 72 + rsave 63, 73 + rsave 64, 74 + rsave 65, 75 + rsave 66, 76 + rsave 67, 77 + rsave 68, 78 + rsave 69, 79 + rsave 70, 80 + rsave 71, 81 + rsave 72, 82 + rsave 73, 83 + rsave 74, 84 + rsave 75, 85 + rsave 76, 86 + rsave 77, 87 + rsave 78, 88 + rsave 79, 89 + rsave 80, 90 + rsave 81, 91 + rsave 82, 92 + rsave 83, 93 + rsave 84, 94 + rsave 85, 95 + rsave 86, 96 + rsave 87, 97 + + /* SAR register */ + rsave 88, 102 + + /* iaoq[0] return address register */ + rsave 89, 100 + .balign 4 +.Lfde0_end: diff --git a/arch/parisc/kernel/vdso32/vdso32.lds.S b/arch/parisc/kernel/vdso32/vdso32.lds.S new file mode 100644 index 000000000000..d4aff3af5262 --- /dev/null +++ b/arch/parisc/kernel/vdso32/vdso32.lds.S @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is the infamous ld script for the 32 bits vdso library + */ +#include <asm/vdso.h> +#include <asm/page.h> + +/* Default link addresses for the vDSOs */ +OUTPUT_FORMAT("elf32-hppa-linux") +OUTPUT_ARCH(hppa) +ENTRY(_start) + +SECTIONS +{ + . = VDSO_LBASE + SIZEOF_HEADERS; + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + . = ALIGN (16); + .text : + { + *(.text .stub .text.* .gnu.linkonce.t.*) + } + PROVIDE (__etext = .); + PROVIDE (_etext = .); + PROVIDE (etext = .); + + /* Other stuff is appended to the text segment: */ + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + .rodata1 : { *(.rodata1) } + .rodata2 : { *(.data.rel.ro) } + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + .gcc_except_table : { *(.gcc_except_table) } + .fixup : { *(.fixup) } + + .dynamic : { *(.dynamic) } :text :dynamic + .plt : { *(.plt) } + .got : { *(.got) } + + _end = .; + __end = .; + PROVIDE (end = .); + + + /* Stabs debugging sections are here too + */ + .stab 0 : { *(.stab) } + .stabstr 0 : { *(.stabstr) } + .stab.excl 0 : { *(.stab.excl) } + .stab.exclstr 0 : { *(.stab.exclstr) } + .stab.index 0 : { *(.stab.index) } + .stab.indexstr 0 : { *(.stab.indexstr) } + .comment 0 : { *(.comment) } + .debug 0 : { *(.debug) } + .line 0 : { *(.line) } + + .debug_srcinfo 0 : { *(.debug_srcinfo) } + .debug_sfnames 0 : { *(.debug_sfnames) } + + .debug_aranges 0 : { *(.debug_aranges) } + .debug_pubnames 0 : { *(.debug_pubnames) } + + .debug_info 0 : { *(.debug_info .gnu.linkonce.wi.*) } + .debug_abbrev 0 : { *(.debug_abbrev) } + .debug_line 0 : { *(.debug_line) } + .debug_frame 0 : { *(.debug_frame) } + .debug_str 0 : { *(.debug_str) } + .debug_loc 0 : { *(.debug_loc) } + .debug_macinfo 0 : { *(.debug_macinfo) } + + .debug_weaknames 0 : { *(.debug_weaknames) } + .debug_funcnames 0 : { *(.debug_funcnames) } + .debug_typenames 0 : { *(.debug_typenames) } + .debug_varnames 0 : { *(.debug_varnames) } + + /DISCARD/ : { *(.note.GNU-stack) } + /DISCARD/ : { *(.data .data.* .gnu.linkonce.d.* .sdata*) } + /DISCARD/ : { *(.bss .sbss .dynbss .dynsbss) } +} + + +PHDRS +{ + text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ + note PT_NOTE FLAGS(4); /* PF_R */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + VDSO_VERSION_STRING { + global: + __kernel_sigtramp_rt32; + __kernel_restart_syscall32; + local: *; + }; +} diff --git a/arch/parisc/kernel/vdso32/vdso32_wrapper.S b/arch/parisc/kernel/vdso32/vdso32_wrapper.S new file mode 100644 index 000000000000..5ac06093e8ec --- /dev/null +++ b/arch/parisc/kernel/vdso32/vdso32_wrapper.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/linkage.h> +#include <asm/page.h> + + __PAGE_ALIGNED_DATA + + .globl vdso32_start, vdso32_end + .balign PAGE_SIZE +vdso32_start: + .incbin "arch/parisc/kernel/vdso32/vdso32.so" + .balign PAGE_SIZE +vdso32_end: + + .previous diff --git a/arch/parisc/kernel/vdso64/Makefile b/arch/parisc/kernel/vdso64/Makefile new file mode 100644 index 000000000000..a30f5ec5eb4b --- /dev/null +++ b/arch/parisc/kernel/vdso64/Makefile @@ -0,0 +1,48 @@ +# List of files in the vdso, has to be asm only for now + +obj-vdso64 = note.o sigtramp.o restart_syscall.o + +# Build rules + +targets := $(obj-vdso64) vdso64.so +obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64)) + + +ccflags-y := -shared -fno-common -fno-builtin +ccflags-y += -nostdlib -Wl,-soname=linux-vdso64.so.1 \ + $(call ld-option, -Wl$(comma)--hash-style=sysv) +asflags-y := -D__VDSO64__ -s + +KBUILD_AFLAGS += -DBUILD_VDSO +KBUILD_CFLAGS += -DBUILD_VDSO -DDISABLE_BRANCH_PROFILING + +VDSO_LIBGCC := $(shell $(CC) -print-libgcc-file-name) + +obj-y += vdso64_wrapper.o +extra-y += vdso64.lds +CPPFLAGS_vdso64.lds += -P -C -U$(ARCH) + +$(obj)/vdso64_wrapper.o : $(obj)/vdso64.so FORCE + +# Force dependency (incbin is bad) +# link rule for the .so file, .lds has to be first +$(obj)/vdso64.so: $(src)/vdso64.lds $(obj-vdso64) $(VDSO_LIBGCC) + $(call if_changed,vdso64ld) + +# assembly rules for the .S files +$(obj-vdso64): %.o: %.S FORCE + $(call if_changed_dep,vdso64as) + +# actual build commands +quiet_cmd_vdso64ld = VDSO64L $@ + cmd_vdso64ld = $(CC) $(c_flags) -Wl,-T $^ -o $@ +quiet_cmd_vdso64as = VDSO64A $@ + cmd_vdso64as = $(CC) $(a_flags) -c -o $@ $< + +# Generate VDSO offsets using helper script +gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +quiet_cmd_vdsosym = VDSOSYM $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + +include/generated/vdso64-offsets.h: $(obj)/vdso64.so FORCE + $(call if_changed,vdsosym) diff --git a/arch/parisc/kernel/vdso64/gen_vdso_offsets.sh b/arch/parisc/kernel/vdso64/gen_vdso_offsets.sh new file mode 100755 index 000000000000..37f05cb38dad --- /dev/null +++ b/arch/parisc/kernel/vdso64/gen_vdso_offsets.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# +# Match symbols in the DSO that look like VDSO_*; produce a header file +# of constant offsets into the shared object. +# +# Doing this inside the Makefile will break the $(filter-out) function, +# causing Kbuild to rebuild the vdso-offsets header file every time. +# +# Inspired by arm64 version. +# + +LC_ALL=C +sed -n 's/\([0-9a-f]*\) . __kernel_\(.*\)/\#define vdso64_offset_\2\t0x\1/p' diff --git a/arch/parisc/kernel/vdso64/note.S b/arch/parisc/kernel/vdso64/note.S new file mode 100644 index 000000000000..bd1fa23597d6 --- /dev/null +++ b/arch/parisc/kernel/vdso64/note.S @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include "../vdso32/note.S" diff --git a/arch/parisc/kernel/vdso64/restart_syscall.S b/arch/parisc/kernel/vdso64/restart_syscall.S new file mode 100644 index 000000000000..83004451f6b1 --- /dev/null +++ b/arch/parisc/kernel/vdso64/restart_syscall.S @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include "../vdso32/restart_syscall.S" diff --git a/arch/parisc/kernel/vdso64/sigtramp.S b/arch/parisc/kernel/vdso64/sigtramp.S new file mode 100644 index 000000000000..66a6d2b241e1 --- /dev/null +++ b/arch/parisc/kernel/vdso64/sigtramp.S @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Signal trampolines for 64 bit processes. + * + * Copyright (C) 2006 Randolph Chung <tausq@debian.org> + * Copyright (C) 2018-2022 Helge Deller <deller@gmx.de> + * Copyright (C) 2022 John David Anglin <dave.anglin@bell.net> + */ +#include <asm/unistd.h> +#include <linux/linkage.h> +#include <generated/asm-offsets.h> + + .text + +/* Gdb expects the trampoline is on the stack and the pc is offset from + a 64-byte boundary by 0, 4 or 5 instructions. Since the vdso trampoline + is not on the stack, we need a new variant with different offsets and + data to tell gdb where to find the signal context on the stack. + + Here we put the offset to the context data at the start of the trampoline + region and offset the first trampoline by 2 instructions. Please do + not change the trampoline as the code in gdb depends on the following + instruction sequence exactly. + */ + .align 64 + .word SIGFRAME_CONTEXT_REGS + +/* The nop here is a hack. The dwarf2 unwind routines subtract 1 from + the return address to get an address in the middle of the presumed + call instruction. Since we don't have a call here, we artifically + extend the range covered by the unwind info by adding a nop before + the real start. + */ + nop + + .globl __kernel_sigtramp_rt + .type __kernel_sigtramp_rt, @function +__kernel_sigtramp_rt: + .proc + .callinfo FRAME=ASM_SIGFRAME_SIZE,CALLS,SAVE_RP + .entry + +.Lsigrt_start = . - 4 +0: ldi 0, %r25 /* (in_syscall=0) */ + ldi __NR_rt_sigreturn, %r20 + ble 0x100(%sr2, %r0) + nop + +1: ldi 1, %r25 /* (in_syscall=1) */ + ldi __NR_rt_sigreturn, %r20 + ble 0x100(%sr2, %r0) + nop +.Lsigrt_end: + .exit + .procend + .size __kernel_sigtramp_rt,.-__kernel_sigtramp_rt + + .section .eh_frame,"a",@progbits + +/* This is where the mcontext_t struct can be found on the stack. */ +#define PTREGS SIGFRAME_CONTEXT_REGS /* 64-bit process offset is -720 */ + +/* Register REGNO can be found at offset OFS of the mcontext_t structure. */ + .macro rsave regno,ofs + .byte 0x05 /* DW_CFA_offset_extended */ + .uleb128 \regno; /* regno */ + .uleb128 \ofs /* factored offset */ + .endm + +.Lcie: + .long .Lcie_end - .Lcie_start +.Lcie_start: + .long 0 /* CIE ID */ + .byte 1 /* Version number */ + .stringz "zRS" /* NUL-terminated augmentation string */ + .uleb128 4 /* Code alignment factor */ + .sleb128 8 /* Data alignment factor */ + .byte 61 /* Return address register column, iaoq[0] */ + .uleb128 1 /* Augmentation value length */ + .byte 0x1b /* DW_EH_PE_pcrel | DW_EH_PE_sdata4. */ + .byte 0x0f /* DW_CFA_def_cfa_expresion */ + .uleb128 9f - 1f /* length */ +1: + .byte 0x8e /* DW_OP_breg30 */ + .sleb128 PTREGS +9: + .balign 8 +.Lcie_end: + + .long .Lfde0_end - .Lfde0_start +.Lfde0_start: + .long .Lfde0_start - .Lcie /* CIE pointer. */ + .long .Lsigrt_start - . /* PC start, length */ + .long .Lsigrt_end - .Lsigrt_start + .uleb128 0 /* Augmentation */ + + /* General registers */ + rsave 1, 2 + rsave 2, 3 + rsave 3, 4 + rsave 4, 5 + rsave 5, 6 + rsave 6, 7 + rsave 7, 8 + rsave 8, 9 + rsave 9, 10 + rsave 10, 11 + rsave 11, 12 + rsave 12, 13 + rsave 13, 14 + rsave 14, 15 + rsave 15, 16 + rsave 16, 17 + rsave 17, 18 + rsave 18, 19 + rsave 19, 20 + rsave 20, 21 + rsave 21, 22 + rsave 22, 23 + rsave 23, 24 + rsave 24, 25 + rsave 25, 26 + rsave 26, 27 + rsave 27, 28 + rsave 28, 29 + rsave 29, 30 + rsave 30, 31 + rsave 31, 32 + + /* Floating-point registers */ + rsave 32, 36 + rsave 33, 37 + rsave 34, 38 + rsave 35, 39 + rsave 36, 40 + rsave 37, 41 + rsave 38, 42 + rsave 39, 43 + rsave 40, 44 + rsave 41, 45 + rsave 42, 46 + rsave 43, 47 + rsave 44, 48 + rsave 45, 49 + rsave 46, 50 + rsave 47, 51 + rsave 48, 52 + rsave 49, 53 + rsave 50, 54 + rsave 51, 55 + rsave 52, 56 + rsave 53, 57 + rsave 54, 58 + rsave 55, 59 + rsave 56, 60 + rsave 57, 61 + rsave 58, 62 + rsave 59, 63 + + /* SAR register */ + rsave 60, 67 + + /* iaoq[0] return address register */ + rsave 61, 65 + .balign 8 +.Lfde0_end: diff --git a/arch/parisc/kernel/vdso64/vdso64.lds.S b/arch/parisc/kernel/vdso64/vdso64.lds.S new file mode 100644 index 000000000000..de1fb4b19286 --- /dev/null +++ b/arch/parisc/kernel/vdso64/vdso64.lds.S @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is the infamous ld script for the 64 bits vdso library + */ +#include <asm/vdso.h> + +/* Default link addresses for the vDSOs */ +OUTPUT_FORMAT("elf64-hppa-linux") +OUTPUT_ARCH(hppa:hppa2.0w) +ENTRY(_start) + +SECTIONS +{ + . = VDSO_LBASE + SIZEOF_HEADERS; + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + . = ALIGN (16); + .text : + { + *(.text .stub .text.* .gnu.linkonce.t.*) + } + PROVIDE (__etext = .); + PROVIDE (_etext = .); + PROVIDE (etext = .); + + /* Other stuff is appended to the text segment: */ + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + .rodata1 : { *(.rodata1) } + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + .gcc_except_table : { *(.gcc_except_table) } + .fixup : { *(.fixup) } + + .dynamic : { *(.dynamic) } :text :dynamic + .plt : { *(.plt) } + .got : { *(.got) } + + _end = .; + __end = .; + PROVIDE (end = .); + + + /* Stabs debugging sections are here too + */ + .stab 0 : { *(.stab) } + .stabstr 0 : { *(.stabstr) } + .stab.excl 0 : { *(.stab.excl) } + .stab.exclstr 0 : { *(.stab.exclstr) } + .stab.index 0 : { *(.stab.index) } + .stab.indexstr 0 : { *(.stab.indexstr) } + .comment 0 : { *(.comment) } + .debug 0 : { *(.debug) } + .line 0 : { *(.line) } + + .debug_srcinfo 0 : { *(.debug_srcinfo) } + .debug_sfnames 0 : { *(.debug_sfnames) } + + .debug_aranges 0 : { *(.debug_aranges) } + .debug_pubnames 0 : { *(.debug_pubnames) } + + .debug_info 0 : { *(.debug_info .gnu.linkonce.wi.*) } + .debug_abbrev 0 : { *(.debug_abbrev) } + .debug_line 0 : { *(.debug_line) } + .debug_frame 0 : { *(.debug_frame) } + .debug_str 0 : { *(.debug_str) } + .debug_loc 0 : { *(.debug_loc) } + .debug_macinfo 0 : { *(.debug_macinfo) } + + .debug_weaknames 0 : { *(.debug_weaknames) } + .debug_funcnames 0 : { *(.debug_funcnames) } + .debug_typenames 0 : { *(.debug_typenames) } + .debug_varnames 0 : { *(.debug_varnames) } + + /DISCARD/ : { *(.note.GNU-stack) } + /DISCARD/ : { *(.data .data.* .gnu.linkonce.d.* .sdata*) } + /DISCARD/ : { *(.bss .sbss .dynbss .dynsbss) } +} + + +PHDRS +{ + text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ + note PT_NOTE FLAGS(4); /* PF_R */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + VDSO_VERSION_STRING { + global: + __kernel_sigtramp_rt64; + __kernel_restart_syscall64; + local: *; + }; +} diff --git a/arch/parisc/kernel/vdso64/vdso64_wrapper.S b/arch/parisc/kernel/vdso64/vdso64_wrapper.S new file mode 100644 index 000000000000..66f929482d3d --- /dev/null +++ b/arch/parisc/kernel/vdso64/vdso64_wrapper.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/linkage.h> +#include <asm/page.h> + + __PAGE_ALIGNED_DATA + + .globl vdso64_start, vdso64_end + .balign PAGE_SIZE +vdso64_start: + .incbin "arch/parisc/kernel/vdso64/vdso64.so" + .balign PAGE_SIZE +vdso64_end: + + .previous diff --git a/arch/parisc/lib/iomap.c b/arch/parisc/lib/iomap.c index 367f6397bda7..860385058085 100644 --- a/arch/parisc/lib/iomap.c +++ b/arch/parisc/lib/iomap.c @@ -346,6 +346,16 @@ u64 ioread64be(const void __iomem *addr) return *((u64 *)addr); } +u64 ioread64_lo_hi(const void __iomem *addr) +{ + u32 low, high; + + low = ioread32(addr); + high = ioread32(addr + sizeof(u32)); + + return low + ((u64)high << 32); +} + u64 ioread64_hi_lo(const void __iomem *addr) { u32 low, high; @@ -419,6 +429,12 @@ void iowrite64be(u64 datum, void __iomem *addr) } } +void iowrite64_lo_hi(u64 val, void __iomem *addr) +{ + iowrite32(val, addr); + iowrite32(val >> 32, addr + sizeof(u32)); +} + void iowrite64_hi_lo(u64 val, void __iomem *addr) { iowrite32(val >> 32, addr + sizeof(u32)); @@ -530,6 +546,7 @@ EXPORT_SYMBOL(ioread32); EXPORT_SYMBOL(ioread32be); EXPORT_SYMBOL(ioread64); EXPORT_SYMBOL(ioread64be); +EXPORT_SYMBOL(ioread64_lo_hi); EXPORT_SYMBOL(ioread64_hi_lo); EXPORT_SYMBOL(iowrite8); EXPORT_SYMBOL(iowrite16); @@ -538,6 +555,7 @@ EXPORT_SYMBOL(iowrite32); EXPORT_SYMBOL(iowrite32be); EXPORT_SYMBOL(iowrite64); EXPORT_SYMBOL(iowrite64be); +EXPORT_SYMBOL(iowrite64_lo_hi); EXPORT_SYMBOL(iowrite64_hi_lo); EXPORT_SYMBOL(ioread8_rep); EXPORT_SYMBOL(ioread16_rep); diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c index ea70a0e08321..5fc0c852c84c 100644 --- a/arch/parisc/lib/memcpy.c +++ b/arch/parisc/lib/memcpy.c @@ -13,8 +13,8 @@ #include <linux/compiler.h> #include <linux/uaccess.h> -#define get_user_space() (uaccess_kernel() ? 0 : mfsp(3)) -#define get_kernel_space() (0) +#define get_user_space() mfsp(SR_USER) +#define get_kernel_space() SR_KERNEL /* Returns 0 for success, otherwise, returns number of bytes not transferred. */ extern unsigned long pa_memcpy(void *dst, const void *src, @@ -23,8 +23,8 @@ extern unsigned long pa_memcpy(void *dst, const void *src, unsigned long raw_copy_to_user(void __user *dst, const void *src, unsigned long len) { - mtsp(get_kernel_space(), 1); - mtsp(get_user_space(), 2); + mtsp(get_kernel_space(), SR_TEMP1); + mtsp(get_user_space(), SR_TEMP2); return pa_memcpy((void __force *)dst, src, len); } EXPORT_SYMBOL(raw_copy_to_user); @@ -32,16 +32,16 @@ EXPORT_SYMBOL(raw_copy_to_user); unsigned long raw_copy_from_user(void *dst, const void __user *src, unsigned long len) { - mtsp(get_user_space(), 1); - mtsp(get_kernel_space(), 2); + mtsp(get_user_space(), SR_TEMP1); + mtsp(get_kernel_space(), SR_TEMP2); return pa_memcpy(dst, (void __force *)src, len); } EXPORT_SYMBOL(raw_copy_from_user); void * memcpy(void * dst,const void *src, size_t count) { - mtsp(get_kernel_space(), 1); - mtsp(get_kernel_space(), 2); + mtsp(get_kernel_space(), SR_TEMP1); + mtsp(get_kernel_space(), SR_TEMP2); pa_memcpy(dst, src, count); return dst; } diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index e9eabf8f14d7..f114e102aaf2 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -425,3 +425,92 @@ out_of_memory: } pagefault_out_of_memory(); } + +/* Handle non-access data TLB miss faults. + * + * For probe instructions, accesses to userspace are considered allowed + * if they lie in a valid VMA and the access type matches. We are not + * allowed to handle MM faults here so there may be situations where an + * actual access would fail even though a probe was successful. + */ +int +handle_nadtlb_fault(struct pt_regs *regs) +{ + unsigned long insn = regs->iir; + int breg, treg, xreg, val = 0; + struct vm_area_struct *vma, *prev_vma; + struct task_struct *tsk; + struct mm_struct *mm; + unsigned long address; + unsigned long acc_type; + + switch (insn & 0x380) { + case 0x280: + /* FDC instruction */ + fallthrough; + case 0x380: + /* PDC and FIC instructions */ + if (printk_ratelimit()) { + pr_warn("BUG: nullifying cache flush/purge instruction\n"); + show_regs(regs); + } + if (insn & 0x20) { + /* Base modification */ + breg = (insn >> 21) & 0x1f; + xreg = (insn >> 16) & 0x1f; + if (breg && xreg) + regs->gr[breg] += regs->gr[xreg]; + } + regs->gr[0] |= PSW_N; + return 1; + + case 0x180: + /* PROBE instruction */ + treg = insn & 0x1f; + if (regs->isr) { + tsk = current; + mm = tsk->mm; + if (mm) { + /* Search for VMA */ + address = regs->ior; + mmap_read_lock(mm); + vma = find_vma_prev(mm, address, &prev_vma); + mmap_read_unlock(mm); + + /* + * Check if access to the VMA is okay. + * We don't allow for stack expansion. + */ + acc_type = (insn & 0x40) ? VM_WRITE : VM_READ; + if (vma + && address >= vma->vm_start + && (vma->vm_flags & acc_type) == acc_type) + val = 1; + } + } + if (treg) + regs->gr[treg] = val; + regs->gr[0] |= PSW_N; + return 1; + + case 0x300: + /* LPA instruction */ + if (insn & 0x20) { + /* Base modification */ + breg = (insn >> 21) & 0x1f; + xreg = (insn >> 16) & 0x1f; + if (breg && xreg) + regs->gr[breg] += regs->gr[xreg]; + } + treg = insn & 0x1f; + if (treg) + regs->gr[treg] = 0; + regs->gr[0] |= PSW_N; + return 1; + + default: + break; + } + + return 0; +} diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index 1ae31db9988f..1dc2e88e7b04 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -337,9 +337,9 @@ static void __init setup_bootmem(void) static bool kernel_set_to_readonly; -static void __init map_pages(unsigned long start_vaddr, - unsigned long start_paddr, unsigned long size, - pgprot_t pgprot, int force) +static void __ref map_pages(unsigned long start_vaddr, + unsigned long start_paddr, unsigned long size, + pgprot_t pgprot, int force) { pmd_t *pmd; pte_t *pg_table; @@ -449,7 +449,7 @@ void __init set_kernel_text_rw(int enable_read_write) flush_tlb_all(); } -void __ref free_initmem(void) +void free_initmem(void) { unsigned long init_begin = (unsigned long)__init_begin; unsigned long init_end = (unsigned long)__init_end; @@ -463,7 +463,6 @@ void __ref free_initmem(void) /* The init text pages are marked R-X. We have to * flush the icache and mark them RW- * - * This is tricky, because map_pages is in the init section. * Do a dummy remap of the data section first (the data * section is already PAGE_KERNEL) to pull in the TLB entries * for map_kernel */ diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index b779603978e1..7e7387bd7d53 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -108,6 +108,7 @@ config PPC select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_HAS_COPY_MC if PPC64 + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX if STRICT_KERNEL_RWX diff --git a/arch/powerpc/configs/linkstation_defconfig b/arch/powerpc/configs/linkstation_defconfig index d4be64f190ff..fa707de761be 100644 --- a/arch/powerpc/configs/linkstation_defconfig +++ b/arch/powerpc/configs/linkstation_defconfig @@ -120,7 +120,6 @@ CONFIG_NFS_V3_ACL=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_932=m diff --git a/arch/powerpc/configs/mvme5100_defconfig b/arch/powerpc/configs/mvme5100_defconfig index 1fed6be95d53..d1c7fd5bf34b 100644 --- a/arch/powerpc/configs/mvme5100_defconfig +++ b/arch/powerpc/configs/mvme5100_defconfig @@ -101,7 +101,6 @@ CONFIG_NFS_V3_ACL=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_CIFS=m CONFIG_NLS=y CONFIG_NLS_CODEPAGE_437=m diff --git a/arch/powerpc/include/asm/agp.h b/arch/powerpc/include/asm/agp.h index b29b1186f819..6b6485c988dd 100644 --- a/arch/powerpc/include/asm/agp.h +++ b/arch/powerpc/include/asm/agp.h @@ -5,8 +5,8 @@ #include <asm/io.h> -#define map_page_into_agp(page) -#define unmap_page_from_agp(page) +#define map_page_into_agp(page) do {} while (0) +#define unmap_page_from_agp(page) do {} while (0) #define flush_agp_cache() mb() /* GATT allocation. Returns/accepts GATT kernel virtual address. */ diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index f8b94f78403f..95e06f2a8e23 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -372,8 +372,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, #define __HAVE_ARCH_PTE_SAME #define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0) -#define pmd_page(pmd) \ - pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) /* * Encode and decode a swap entry. diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index ba5b1becf518..006cbec70ffe 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -202,7 +202,6 @@ static inline struct subpage_prot_table *mm_ctx_subpage_prot(mm_context_t *ctx) /* * The current system page and segment sizes */ -extern int mmu_linear_psize; extern int mmu_virtual_psize; extern int mmu_vmalloc_psize; extern int mmu_io_psize; @@ -213,6 +212,7 @@ extern int mmu_io_psize; #define mmu_virtual_psize MMU_PAGE_4K #endif #endif +extern int mmu_linear_psize; extern int mmu_vmemmap_psize; /* MMU initialization */ diff --git a/arch/powerpc/include/asm/fadump-internal.h b/arch/powerpc/include/asm/fadump-internal.h index 52189928ec08..81bcb9abb371 100644 --- a/arch/powerpc/include/asm/fadump-internal.h +++ b/arch/powerpc/include/asm/fadump-internal.h @@ -19,11 +19,6 @@ #define memblock_num_regions(memblock_type) (memblock.memblock_type.cnt) -/* Alignment per CMA requirement. */ -#define FADUMP_CMA_ALIGNMENT (PAGE_SIZE << \ - max_t(unsigned long, MAX_ORDER - 1, \ - pageblock_order)) - /* FAD commands */ #define FADUMP_REGISTER 1 #define FADUMP_UNREGISTER 2 diff --git a/arch/powerpc/include/asm/kexec_ranges.h b/arch/powerpc/include/asm/kexec_ranges.h index 7a90000f8d15..f83866a19e87 100644 --- a/arch/powerpc/include/asm/kexec_ranges.h +++ b/arch/powerpc/include/asm/kexec_ranges.h @@ -9,7 +9,7 @@ struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges); int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size); int add_tce_mem_ranges(struct crash_mem **mem_ranges); int add_initrd_mem_range(struct crash_mem **mem_ranges); -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU int add_htab_mem_range(struct crash_mem **mem_ranges); #else static inline int add_htab_mem_range(struct crash_mem **mem_ranges) diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index fd277b15635c..b8527a74bd4d 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -21,7 +21,6 @@ extern void destroy_context(struct mm_struct *mm); #ifdef CONFIG_SPAPR_TCE_IOMMU struct mm_iommu_table_group_mem_t; -extern int isolate_lru_page(struct page *page); /* from internal.h */ extern bool mm_iommu_preregistered(struct mm_struct *mm); extern long mm_iommu_new(struct mm_struct *mm, unsigned long ua, unsigned long entries, diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index 160abcb8e9fa..ea0e487f87b1 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -9,7 +9,7 @@ long soft_nmi_interrupt(struct pt_regs *regs); static inline void arch_touch_nmi_watchdog(void) {} #endif -#if defined(CONFIG_NMI_IPI) && defined(CONFIG_STACKTRACE) +#ifdef CONFIG_NMI_IPI extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h index 64b6c608eca4..de092b04ee1a 100644 --- a/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h @@ -71,9 +71,9 @@ static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags size_t size = 1UL << shift; if (size == SZ_16K) - return __pte(pte_val(entry) & ~_PAGE_HUGE); + return __pte(pte_val(entry) | _PAGE_SPS); else - return entry; + return __pte(pte_val(entry) | _PAGE_SPS | _PAGE_HUGE); } #define arch_make_huge_pte arch_make_huge_pte #endif diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index d959c2a73fbf..dcc9b338e042 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -349,15 +349,14 @@ static inline int pte_young(pte_t pte) * of the pte page. -- paulus */ #ifndef CONFIG_BOOKE -#define pmd_page(pmd) \ - pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) #else #define pmd_page_vaddr(pmd) \ ((unsigned long)(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1))) -#define pmd_page(pmd) \ - pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT)) +#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) #endif +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) /* * Encode and decode a swap entry. * Note that the bits we use in a PTE for representing a swap entry diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 2816d158280a..78888b0c30f6 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -142,6 +142,7 @@ static inline pte_t pmd_pte(pmd_t pmd) #define pmd_present(pmd) (!pmd_none(pmd)) #define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) extern struct page *pmd_page(pmd_t pmd); +#define pmd_pfn(pmd) (page_to_pfn(pmd_page(pmd))) static inline void pud_set(pud_t *pudp, unsigned long val) { diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 63316100080c..2e83217f52de 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -11,18 +11,9 @@ #ifdef __powerpc64__ /* We use TASK_SIZE_USER64 as TASK_SIZE is not constant */ #define TASK_SIZE_MAX TASK_SIZE_USER64 -#else -#define TASK_SIZE_MAX TASK_SIZE #endif -static inline bool __access_ok(unsigned long addr, unsigned long size) -{ - return addr < TASK_SIZE_MAX && size <= TASK_SIZE_MAX - addr; -} - -#define access_ok(addr, size) \ - (__chk_user_ptr(addr), \ - __access_ok((unsigned long)(addr), (size))) +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They automatically @@ -467,8 +458,6 @@ do { \ unsafe_put_user(*(u8*)(_src + _i), (u8 __user *)(_dst + _i), e); \ } while (0) -#define HAVE_GET_KERNEL_NOFAULT - #define __get_kernel_nofault(dst, src, type, err_label) \ __get_user_size_goto(*((type *)(dst)), \ (__force type __user *)(src), sizeof(type), err_label) diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h index 6ca923510b59..294620a25f80 100644 --- a/arch/powerpc/include/asm/xor_altivec.h +++ b/arch/powerpc/include/asm/xor_altivec.h @@ -3,17 +3,20 @@ #define _ASM_POWERPC_XOR_ALTIVEC_H #ifdef CONFIG_ALTIVEC - -void xor_altivec_2(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in); -void xor_altivec_3(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in); -void xor_altivec_4(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in); -void xor_altivec_5(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in, unsigned long *v5_in); +void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); #endif #endif /* _ASM_POWERPC_XOR_ALTIVEC_H */ diff --git a/arch/powerpc/include/uapi/asm/shmbuf.h b/arch/powerpc/include/uapi/asm/shmbuf.h index 00422b2f3c63..439a3a02ba64 100644 --- a/arch/powerpc/include/uapi/asm/shmbuf.h +++ b/arch/powerpc/include/uapi/asm/shmbuf.h @@ -2,6 +2,9 @@ #ifndef _ASM_POWERPC_SHMBUF_H #define _ASM_POWERPC_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -34,7 +37,7 @@ struct shmid64_ds { unsigned long shm_ctime; /* last change time */ unsigned long __unused4; #endif - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ __kernel_pid_t shm_cpid; /* pid of creator */ __kernel_pid_t shm_lpid; /* pid of last operator */ unsigned long shm_nattch; /* no. of current attaches */ diff --git a/arch/powerpc/include/uapi/asm/signal.h b/arch/powerpc/include/uapi/asm/signal.h index 04873dd311c2..37d41d87c45b 100644 --- a/arch/powerpc/include/uapi/asm/signal.h +++ b/arch/powerpc/include/uapi/asm/signal.h @@ -86,7 +86,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index d03e488cfe9c..4fdb7c77fda1 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -113,6 +113,12 @@ static int __init fadump_cma_init(void) } /* + * If CMA activation fails, keep the pages reserved, instead of + * exposing them to buddy allocator. Same as 'fadump=nocma' case. + */ + cma_reserve_pages_on_error(fadump_cma); + + /* * So we now have successfully initialized cma area for fadump. */ pr_info("Initialized 0x%lx bytes cma area at %ldMB from 0x%lx " @@ -544,7 +550,7 @@ int __init fadump_reserve_mem(void) if (!fw_dump.nocma) { fw_dump.boot_memory_size = ALIGN(fw_dump.boot_memory_size, - FADUMP_CMA_ALIGNMENT); + CMA_MIN_ALIGNMENT_BYTES); } #endif diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index fa84744d6b24..b876ef8c70a7 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -421,14 +421,14 @@ InstructionTLBMiss: */ /* Get PTE (linux-style) and check access */ mfspr r3,SPRN_IMISS -#ifdef CONFIG_MODULES +#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) lis r1, TASK_SIZE@h /* check if kernel address */ cmplw 0,r1,r3 #endif mfspr r2, SPRN_SDR1 li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC | _PAGE_USER rlwinm r2, r2, 28, 0xfffff000 -#ifdef CONFIG_MODULES +#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) bgt- 112f lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */ li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index d45a415d5374..2069bbb90a9a 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -1110,14 +1110,6 @@ EXPORT_SYMBOL_GPL(cpu_remove_dev_attr_group); /* NUMA stuff */ #ifdef CONFIG_NUMA -static void __init register_nodes(void) -{ - int i; - - for (i = 0; i < MAX_NUMNODES; i++) - register_one_node(i); -} - int sysfs_add_device_to_node(struct device *dev, int nid) { struct node *node = node_devices[nid]; @@ -1132,13 +1124,6 @@ void sysfs_remove_device_from_node(struct device *dev, int nid) sysfs_remove_link(&node->dev.kobj, kobject_name(&dev->kobj)); } EXPORT_SYMBOL_GPL(sysfs_remove_device_from_node); - -#else -static void __init register_nodes(void) -{ - return; -} - #endif /* Only valid if CPU is present. */ @@ -1155,8 +1140,6 @@ static int __init topology_init(void) { int cpu, r; - register_nodes(); - for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index e414ca44839f..8cabdb39cbbc 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -91,6 +91,7 @@ #include <linux/kvm_host.h> #include <linux/ksm.h> #include <linux/of.h> +#include <linux/memremap.h> #include <asm/ultravisor.h> #include <asm/mman.h> #include <asm/kvm_ppc.h> @@ -712,7 +713,6 @@ static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) dpage = pfn_to_page(uvmem_pfn); dpage->zone_device_data = pvt; - get_page(dpage); lock_page(dpage); return dpage; out_clear: diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index a94b0cd0bdc5..5ed88fbee0db 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -112,9 +112,9 @@ static nokprobe_inline long address_ok(struct pt_regs *regs, { if (!user_mode(regs)) return 1; - if (__access_ok(ea, nb)) + if (access_ok((void __user *)ea, nb)) return 1; - if (__access_ok(ea, 1)) + if (access_ok((void __user *)ea, 1)) /* Access overlaps the end of the user region */ regs->dar = TASK_SIZE_MAX - 1; else @@ -3264,12 +3264,14 @@ void emulate_update_regs(struct pt_regs *regs, struct instruction_op *op) case BARRIER_EIEIO: eieio(); break; +#ifdef CONFIG_PPC64 case BARRIER_LWSYNC: asm volatile("lwsync" : : : "memory"); break; case BARRIER_PTESYNC: asm volatile("ptesync" : : : "memory"); break; +#endif } break; diff --git a/arch/powerpc/lib/xor_vmx.c b/arch/powerpc/lib/xor_vmx.c index 54e61979e80e..aab49d056d18 100644 --- a/arch/powerpc/lib/xor_vmx.c +++ b/arch/powerpc/lib/xor_vmx.c @@ -49,8 +49,9 @@ typedef vector signed char unative_t; V1##_3 = vec_xor(V1##_3, V2##_3); \ } while (0) -void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in) +void __xor_altivec_2(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in) { DEFINE(v1); DEFINE(v2); @@ -67,8 +68,10 @@ void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in, } while (--lines > 0); } -void __xor_altivec_3(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in) +void __xor_altivec_3(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in) { DEFINE(v1); DEFINE(v2); @@ -89,9 +92,11 @@ void __xor_altivec_3(unsigned long bytes, unsigned long *v1_in, } while (--lines > 0); } -void __xor_altivec_4(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in) +void __xor_altivec_4(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in, + const unsigned long * __restrict v4_in) { DEFINE(v1); DEFINE(v2); @@ -116,9 +121,12 @@ void __xor_altivec_4(unsigned long bytes, unsigned long *v1_in, } while (--lines > 0); } -void __xor_altivec_5(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in, unsigned long *v5_in) +void __xor_altivec_5(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in, + const unsigned long * __restrict v4_in, + const unsigned long * __restrict v5_in) { DEFINE(v1); DEFINE(v2); diff --git a/arch/powerpc/lib/xor_vmx.h b/arch/powerpc/lib/xor_vmx.h index 5c2b0839b179..573c41d90dac 100644 --- a/arch/powerpc/lib/xor_vmx.h +++ b/arch/powerpc/lib/xor_vmx.h @@ -6,16 +6,17 @@ * outside of the enable/disable altivec block. */ -void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in); - -void __xor_altivec_3(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in); - -void __xor_altivec_4(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in); - -void __xor_altivec_5(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in, unsigned long *v5_in); +void __xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void __xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void __xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void __xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); diff --git a/arch/powerpc/lib/xor_vmx_glue.c b/arch/powerpc/lib/xor_vmx_glue.c index 80dba916c367..35d917ece4d1 100644 --- a/arch/powerpc/lib/xor_vmx_glue.c +++ b/arch/powerpc/lib/xor_vmx_glue.c @@ -12,47 +12,51 @@ #include <asm/xor_altivec.h> #include "xor_vmx.h" -void xor_altivec_2(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in) +void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { preempt_disable(); enable_kernel_altivec(); - __xor_altivec_2(bytes, v1_in, v2_in); + __xor_altivec_2(bytes, p1, p2); disable_kernel_altivec(); preempt_enable(); } EXPORT_SYMBOL(xor_altivec_2); -void xor_altivec_3(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in) +void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { preempt_disable(); enable_kernel_altivec(); - __xor_altivec_3(bytes, v1_in, v2_in, v3_in); + __xor_altivec_3(bytes, p1, p2, p3); disable_kernel_altivec(); preempt_enable(); } EXPORT_SYMBOL(xor_altivec_3); -void xor_altivec_4(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in) +void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { preempt_disable(); enable_kernel_altivec(); - __xor_altivec_4(bytes, v1_in, v2_in, v3_in, v4_in); + __xor_altivec_4(bytes, p1, p2, p3, p4); disable_kernel_altivec(); preempt_enable(); } EXPORT_SYMBOL(xor_altivec_4); -void xor_altivec_5(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in, unsigned long *v3_in, - unsigned long *v4_in, unsigned long *v5_in) +void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { preempt_disable(); enable_kernel_altivec(); - __xor_altivec_5(bytes, v1_in, v2_in, v3_in, v4_in, v5_in); + __xor_altivec_5(bytes, p1, p2, p3, p4, p5); disable_kernel_altivec(); preempt_enable(); } diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 79ce3c22a29d..052e6590f84f 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -6,6 +6,7 @@ #include <linux/sched.h> #include <linux/mm_types.h> #include <linux/memblock.h> +#include <linux/memremap.h> #include <linux/debugfs.h> #include <misc/cxl-base.h> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5adcbd9b5e88..0804b9a11934 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -40,6 +40,7 @@ config RISCV select ARCH_USE_MEMTEST select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS + select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU @@ -171,9 +172,6 @@ config ARCH_SPARSEMEM_ENABLE config ARCH_SELECT_MEMORY_MODEL def_bool ARCH_SPARSEMEM_ENABLE -config ARCH_WANT_GENERAL_HUGETLB - def_bool y - config ARCH_SUPPORTS_UPROBES def_bool y diff --git a/arch/riscv/Kconfig.erratas b/arch/riscv/Kconfig.erratas index b44d6ecdb46e..0aacd7052585 100644 --- a/arch/riscv/Kconfig.erratas +++ b/arch/riscv/Kconfig.erratas @@ -2,6 +2,7 @@ menu "CPU errata selection" config RISCV_ERRATA_ALTERNATIVE bool "RISC-V alternative scheme" + depends on !XIP_KERNEL default y help This Kconfig allows the kernel to automatically patch the diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 6ec44a22278a..c112ab2a9052 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -14,8 +14,8 @@ config SOC_SIFIVE select CLK_SIFIVE select CLK_SIFIVE_PRCI select SIFIVE_PLIC - select RISCV_ERRATA_ALTERNATIVE - select ERRATA_SIFIVE + select RISCV_ERRATA_ALTERNATIVE if !XIP_KERNEL + select ERRATA_SIFIVE if !XIP_KERNEL help This enables support for SiFive SoC platform hardware. diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 8a107ed18b0d..7d81102cffd4 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -50,6 +50,12 @@ riscv-march-$(CONFIG_ARCH_RV32I) := rv32ima riscv-march-$(CONFIG_ARCH_RV64I) := rv64ima riscv-march-$(CONFIG_FPU) := $(riscv-march-y)fd riscv-march-$(CONFIG_RISCV_ISA_C) := $(riscv-march-y)c + +# Newer binutils versions default to ISA spec version 20191213 which moves some +# instructions from the I extension to the Zicsr and Zifencei extensions. +toolchain-need-zicsr-zifencei := $(call cc-option-yn, -march=$(riscv-march-y)_zicsr_zifencei) +riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei + KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y)) KBUILD_AFLAGS += -march=$(riscv-march-y) diff --git a/arch/riscv/boot/dts/canaan/k210.dtsi b/arch/riscv/boot/dts/canaan/k210.dtsi index 56f57118c633..44d338514761 100644 --- a/arch/riscv/boot/dts/canaan/k210.dtsi +++ b/arch/riscv/boot/dts/canaan/k210.dtsi @@ -113,7 +113,8 @@ compatible = "canaan,k210-plic", "sifive,plic-1.0.0"; reg = <0xC000000 0x4000000>; interrupt-controller; - interrupts-extended = <&cpu0_intc 11>, <&cpu1_intc 11>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>; riscv,ndev = <65>; }; diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index 2a82a3b2992b..af64b95e88cc 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -23,7 +23,7 @@ CONFIG_SLOB=y CONFIG_SOC_CANAAN=y CONFIG_SMP=y CONFIG_NR_CPUS=2 -CONFIG_CMDLINE="earlycon console=ttySIF0 rootdelay=2 root=/dev/mmcblk0p1 ro" +CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro" CONFIG_CMDLINE_FORCE=y # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 160e3a1e8f8b..004372f8da54 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -119,7 +119,7 @@ extern phys_addr_t phys_ram_base; ((x) >= kernel_map.virt_addr && (x) < (kernel_map.virt_addr + kernel_map.size)) #define is_linear_mapping(x) \ - ((x) >= PAGE_OFFSET && (!IS_ENABLED(CONFIG_64BIT) || (x) < kernel_map.virt_addr)) + ((x) >= PAGE_OFFSET && (!IS_ENABLED(CONFIG_64BIT) || (x) < PAGE_OFFSET + KERN_VIRT_SIZE)) #define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) + kernel_map.va_pa_offset)) #define kernel_mapping_pa_to_va(y) ({ \ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 7e949f25c933..e3549e50de95 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -13,6 +13,7 @@ #ifndef CONFIG_MMU #define KERNEL_LINK_ADDR PAGE_OFFSET +#define KERN_VIRT_SIZE (UL(-1)) #else #define ADDRESS_SPACE_END (UL(-1)) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index c701a5e57a2b..855450bed9f5 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -21,42 +21,13 @@ #include <asm/byteorder.h> #include <asm/extable.h> #include <asm/asm.h> +#include <asm-generic/access_ok.h> #define __enable_user_access() \ __asm__ __volatile__ ("csrs sstatus, %0" : : "r" (SR_SUM) : "memory") #define __disable_user_access() \ __asm__ __volatile__ ("csrc sstatus, %0" : : "r" (SR_SUM) : "memory") -/** - * access_ok: - Checks if a user space pointer is valid - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block may be valid, false (zero) - * if it is definitely invalid. - * - * Note that, depending on architecture, this function probably just - * checks that the pointer is in the user space range - after calling - * this function, memory access functions may still return -EFAULT. - */ -#define access_ok(addr, size) ({ \ - __chk_user_ptr(addr); \ - likely(__access_ok((unsigned long __force)(addr), (size))); \ -}) - -/* - * Ensure that the range [addr, addr+size) is within the process's - * address space - */ -static inline int __access_ok(unsigned long addr, unsigned long size) -{ - return size <= TASK_SIZE && addr <= TASK_SIZE - size; -} - /* * The exception table consists of pairs of addresses: the first is the * address of an instruction that is allowed to fault, and the second is @@ -346,8 +317,6 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __clear_user(to, n) : n; } -#define HAVE_GET_KERNEL_NOFAULT - #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ long __kr_err; \ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 612556faa527..ffc87e76b1dd 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -51,6 +51,8 @@ obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o +obj-$(CONFIG_TRACE_IRQFLAGS) += trace_irq.o + obj-$(CONFIG_RISCV_BASE_PMU) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index be7f05b542bb..f7a832e3a1d1 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -12,6 +12,7 @@ #include <linux/sched/hotplug.h> #include <asm/irq.h> #include <asm/cpu_ops.h> +#include <asm/numa.h> #include <asm/sbi.h> bool cpu_has_hotplug(unsigned int cpu) @@ -40,6 +41,7 @@ int __cpu_disable(void) return ret; remove_cpu_topology(cpu); + numa_remove_cpu(cpu); set_cpu_online(cpu, false); irq_migrate_all_off_this_cpu(); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index ed29e9c8f660..d6a46ed0bf05 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -108,7 +108,7 @@ _save_context: .option pop #ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off + call __trace_hardirqs_off #endif #ifdef CONFIG_CONTEXT_TRACKING @@ -143,7 +143,7 @@ skip_context_tracking: li t0, EXC_BREAKPOINT beq s4, t0, 1f #ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_on + call __trace_hardirqs_on #endif csrs CSR_STATUS, SR_IE @@ -234,7 +234,7 @@ ret_from_exception: REG_L s0, PT_STATUS(sp) csrc CSR_STATUS, SR_IE #ifdef CONFIG_TRACE_IRQFLAGS - call trace_hardirqs_off + call __trace_hardirqs_off #endif #ifdef CONFIG_RISCV_M_MODE /* the MPP value is too large to be used as an immediate arg for addi */ @@ -270,10 +270,10 @@ restore_all: REG_L s1, PT_STATUS(sp) andi t0, s1, SR_PIE beqz t0, 1f - call trace_hardirqs_on + call __trace_hardirqs_on j 2f 1: - call trace_hardirqs_off + call __trace_hardirqs_off 2: #endif REG_L a0, PT_STATUS(sp) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 2363b43312fc..ec07f991866a 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -22,14 +22,13 @@ add \reg, \reg, t0 .endm .macro XIP_FIXUP_FLASH_OFFSET reg - la t1, __data_loc - li t0, XIP_OFFSET_MASK - and t1, t1, t0 - li t1, XIP_OFFSET - sub t0, t0, t1 - sub \reg, \reg, t0 + la t0, __data_loc + REG_L t1, _xip_phys_offset + sub \reg, \reg, t1 + add \reg, \reg, t0 .endm _xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET +_xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET #else .macro XIP_FIXUP_OFFSET reg .endm diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 68a9e3d1fe16..4a48287513c3 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -13,6 +13,19 @@ #include <linux/pgtable.h> #include <asm/sections.h> +/* + * The auipc+jalr instruction pair can reach any PC-relative offset + * in the range [-2^31 - 2^11, 2^31 - 2^11) + */ +static bool riscv_insn_valid_32bit_offset(ptrdiff_t val) +{ +#ifdef CONFIG_32BIT + return true; +#else + return (-(1L << 31) - (1L << 11)) <= val && val < ((1L << 31) - (1L << 11)); +#endif +} + static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v) { if (v != (u32)v) { @@ -95,7 +108,7 @@ static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location, ptrdiff_t offset = (void *)v - (void *)location; s32 hi20; - if (offset != (s32)offset) { + if (!riscv_insn_valid_32bit_offset(offset)) { pr_err( "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", me->name, (long long)v, location); @@ -197,10 +210,9 @@ static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, Elf_Addr v) { ptrdiff_t offset = (void *)v - (void *)location; - s32 fill_v = offset; u32 hi20, lo12; - if (offset != fill_v) { + if (!riscv_insn_valid_32bit_offset(offset)) { /* Only emit the plt entry if offset over 32-bit range */ if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) { offset = module_emit_plt_entry(me, v); @@ -224,10 +236,9 @@ static int apply_r_riscv_call_rela(struct module *me, u32 *location, Elf_Addr v) { ptrdiff_t offset = (void *)v - (void *)location; - s32 fill_v = offset; u32 hi20, lo12; - if (offset != fill_v) { + if (!riscv_insn_valid_32bit_offset(offset)) { pr_err( "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", me->name, (long long)v, location); diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 1fc075b8f764..55faa4991b87 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -15,8 +15,8 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, { struct stackframe buftail; unsigned long ra = 0; - unsigned long *user_frame_tail = - (unsigned long *)(fp - sizeof(struct stackframe)); + unsigned long __user *user_frame_tail = + (unsigned long __user *)(fp - sizeof(struct stackframe)); /* Check accessibility of one struct frame_tail beyond */ if (!access_ok(user_frame_tail, sizeof(buftail))) diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index f72527fcb347..775d3322b422 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -5,6 +5,7 @@ * Copyright (c) 2020 Western Digital Corporation or its affiliates. */ +#include <linux/bits.h> #include <linux/init.h> #include <linux/pm.h> #include <linux/reboot.h> @@ -85,7 +86,7 @@ static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mas pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n"); break; } - hmask |= 1 << hartid; + hmask |= BIT(hartid); } return hmask; @@ -160,7 +161,7 @@ static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask) { unsigned long hart_mask; - if (!cpu_mask) + if (!cpu_mask || cpumask_empty(cpu_mask)) cpu_mask = cpu_online_mask; hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); @@ -176,7 +177,7 @@ static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, int result = 0; unsigned long hart_mask; - if (!cpu_mask) + if (!cpu_mask || cpumask_empty(cpu_mask)) cpu_mask = cpu_online_mask; hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); @@ -249,26 +250,37 @@ static void __sbi_set_timer_v02(uint64_t stime_value) static int __sbi_send_ipi_v02(const struct cpumask *cpu_mask) { - unsigned long hartid, cpuid, hmask = 0, hbase = 0; + unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; struct sbiret ret = {0}; int result; - if (!cpu_mask) + if (!cpu_mask || cpumask_empty(cpu_mask)) cpu_mask = cpu_online_mask; for_each_cpu(cpuid, cpu_mask) { hartid = cpuid_to_hartid_map(cpuid); - if (hmask && ((hbase + BITS_PER_LONG) <= hartid)) { - ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, - hmask, hbase, 0, 0, 0, 0); - if (ret.error) - goto ecall_failed; - hmask = 0; - hbase = 0; + if (hmask) { + if (hartid + BITS_PER_LONG <= htop || + hbase + BITS_PER_LONG <= hartid) { + ret = sbi_ecall(SBI_EXT_IPI, + SBI_EXT_IPI_SEND_IPI, hmask, + hbase, 0, 0, 0, 0); + if (ret.error) + goto ecall_failed; + hmask = 0; + } else if (hartid < hbase) { + /* shift the mask to fit lower hartid */ + hmask <<= hbase - hartid; + hbase = hartid; + } } - if (!hmask) + if (!hmask) { hbase = hartid; - hmask |= 1UL << (hartid - hbase); + htop = hartid; + } else if (hartid > htop) { + htop = hartid; + } + hmask |= BIT(hartid - hbase); } if (hmask) { @@ -344,25 +356,35 @@ static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) { - unsigned long hartid, cpuid, hmask = 0, hbase = 0; + unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; int result; - if (!cpu_mask) + if (!cpu_mask || cpumask_empty(cpu_mask)) cpu_mask = cpu_online_mask; for_each_cpu(cpuid, cpu_mask) { hartid = cpuid_to_hartid_map(cpuid); - if (hmask && ((hbase + BITS_PER_LONG) <= hartid)) { - result = __sbi_rfence_v02_call(fid, hmask, hbase, - start, size, arg4, arg5); - if (result) - return result; - hmask = 0; - hbase = 0; + if (hmask) { + if (hartid + BITS_PER_LONG <= htop || + hbase + BITS_PER_LONG <= hartid) { + result = __sbi_rfence_v02_call(fid, hmask, + hbase, start, size, arg4, arg5); + if (result) + return result; + hmask = 0; + } else if (hartid < hbase) { + /* shift the mask to fit lower hartid */ + hmask <<= hbase - hartid; + hbase = hartid; + } } - if (!hmask) + if (!hmask) { hbase = hartid; - hmask |= 1UL << (hartid - hbase); + htop = hartid; + } else if (hartid > htop) { + htop = hartid; + } + hmask |= BIT(hartid - hbase); } if (hmask) { diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index b42bfdc67482..834eb652a7b9 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -301,9 +301,6 @@ static int __init topology_init(void) { int i, ret; - for_each_online_node(i) - register_one_node(i); - for_each_possible_cpu(i) { struct cpu *cpu = &per_cpu(cpu_devices, i); diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 201ee206fb57..14d2b53ec322 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -22,15 +22,16 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, bool (*fn)(void *, unsigned long), void *arg) { unsigned long fp, sp, pc; + int level = 0; if (regs) { fp = frame_pointer(regs); sp = user_stack_pointer(regs); pc = instruction_pointer(regs); } else if (task == NULL || task == current) { - fp = (unsigned long)__builtin_frame_address(1); - sp = (unsigned long)__builtin_frame_address(0); - pc = (unsigned long)__builtin_return_address(0); + fp = (unsigned long)__builtin_frame_address(0); + sp = sp_in_global; + pc = (unsigned long)walk_stackframe; } else { /* task blocked in __switch_to */ fp = task->thread.s[0]; @@ -42,7 +43,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, unsigned long low, high; struct stackframe *frame; - if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc))) + if (unlikely(!__kernel_text_address(pc) || (level++ >= 1 && !fn(arg, pc)))) break; /* Validate frame pointer */ diff --git a/arch/riscv/kernel/trace_irq.c b/arch/riscv/kernel/trace_irq.c new file mode 100644 index 000000000000..095ac976d7da --- /dev/null +++ b/arch/riscv/kernel/trace_irq.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com> + */ + +#include <linux/irqflags.h> +#include <linux/kprobes.h> +#include "trace_irq.h" + +/* + * trace_hardirqs_on/off require the caller to setup frame pointer properly. + * Otherwise, CALLER_ADDR1 might trigger an pagging exception in kernel. + * Here we add one extra level so they can be safely called by low + * level entry code which $fp is used for other purpose. + */ + +void __trace_hardirqs_on(void) +{ + trace_hardirqs_on(); +} +NOKPROBE_SYMBOL(__trace_hardirqs_on); + +void __trace_hardirqs_off(void) +{ + trace_hardirqs_off(); +} +NOKPROBE_SYMBOL(__trace_hardirqs_off); diff --git a/arch/riscv/kernel/trace_irq.h b/arch/riscv/kernel/trace_irq.h new file mode 100644 index 000000000000..99fe67377e5e --- /dev/null +++ b/arch/riscv/kernel/trace_irq.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com> + */ +#ifndef __TRACE_IRQ_H +#define __TRACE_IRQ_H + +void __trace_hardirqs_on(void); +void __trace_hardirqs_off(void); + +#endif /* __TRACE_IRQ_H */ diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 7ebaef10ea1b..ac7a25298a04 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -24,6 +24,9 @@ obj-$(CONFIG_KASAN) += kasan_init.o ifdef CONFIG_KASAN KASAN_SANITIZE_kasan_init.o := n KASAN_SANITIZE_init.o := n +ifdef CONFIG_DEBUG_VIRTUAL +KASAN_SANITIZE_physaddr.o := n +endif endif obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 05978f78579f..35484d830fd6 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -33,7 +33,7 @@ static inline void regs_set_gpr(struct pt_regs *regs, unsigned int offset, if (unlikely(offset > MAX_REG_OFFSET)) return; - if (!offset) + if (offset) *(unsigned long *)((unsigned long)regs + offset) = val; } @@ -43,8 +43,8 @@ static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data); - regs_set_gpr(regs, reg_err, -EFAULT); - regs_set_gpr(regs, reg_zero, 0); + regs_set_gpr(regs, reg_err * sizeof(unsigned long), -EFAULT); + regs_set_gpr(regs, reg_zero * sizeof(unsigned long), 0); regs->epc = get_ex_fixup(ex); return true; diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index cf4d018b7d66..0d588032d6e6 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -125,7 +125,6 @@ void __init mem_init(void) else swiotlb_force = SWIOTLB_NO_FORCE; #endif - high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); memblock_free_all(); print_vm_layout(); @@ -195,6 +194,7 @@ static void __init setup_bootmem(void) min_low_pfn = PFN_UP(phys_ram_base); max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); + high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); @@ -232,6 +232,7 @@ static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAG #ifdef CONFIG_XIP_KERNEL #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) +#define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base)) #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) @@ -522,6 +523,7 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) } #ifdef CONFIG_XIP_KERNEL +#define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) extern char _xiprom[], _exiprom[], __data_loc; /* called from head.S with MMU off */ diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index f61f7ca6fe0f..cd1a145257b7 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -113,8 +113,11 @@ static void __init kasan_populate_pud(pgd_t *pgd, base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd))); } else { base_pud = (pud_t *)pgd_page_vaddr(*pgd); - if (base_pud == lm_alias(kasan_early_shadow_pud)) + if (base_pud == lm_alias(kasan_early_shadow_pud)) { base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); + memcpy(base_pud, (void *)kasan_early_shadow_pud, + sizeof(pud_t) * PTRS_PER_PUD); + } } pudp = base_pud + pud_index(vaddr); @@ -202,8 +205,7 @@ asmlinkage void __init kasan_early_init(void) for (i = 0; i < PTRS_PER_PTE; ++i) set_pte(kasan_early_shadow_pte + i, - mk_pte(virt_to_page(kasan_early_shadow_page), - PAGE_KERNEL)); + pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL)); for (i = 0; i < PTRS_PER_PMD; ++i) set_pmd(kasan_early_shadow_pmd + i, diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c index e7fd0c253c7b..19cf25a74ee2 100644 --- a/arch/riscv/mm/physaddr.c +++ b/arch/riscv/mm/physaddr.c @@ -8,12 +8,10 @@ phys_addr_t __virt_to_phys(unsigned long x) { - phys_addr_t y = x - PAGE_OFFSET; - /* * Boundary checking aginst the kernel linear mapping space. */ - WARN(y >= KERN_VIRT_SIZE, + WARN(!is_linear_mapping(x) && !is_kernel_mapping(x), "virt_to_phys used for non-linear address: %pK (%pS)\n", (void *)x, (void *)x); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index be9f39fd06df..2e11ef97c8b9 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -55,11 +55,13 @@ config S390 # Note: keep this list sorted alphabetically # imply IMA_SECURE_AND_OR_TRUSTED_BOOT + select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_USTAT_F_TINODE select ARCH_BINFMT_ELF_STATE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX select ARCH_HAS_DEVMEM_IS_ALLOWED diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h index 16dc57dd90b3..8511f0e59290 100644 --- a/arch/s390/include/asm/extable.h +++ b/arch/s390/include/asm/extable.h @@ -69,8 +69,13 @@ static inline void swap_ex_entry_fixup(struct exception_table_entry *a, { a->fixup = b->fixup + delta; b->fixup = tmp.fixup - delta; - a->handler = b->handler + delta; - b->handler = tmp.handler - delta; + a->handler = b->handler; + if (a->handler) + a->handler += delta; + b->handler = tmp.handler; + if (b->handler) + b->handler -= delta; } +#define swap_ex_entry_fixup swap_ex_entry_fixup #endif diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 267f70f4393f..6f80ec9c04be 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -47,15 +47,17 @@ struct ftrace_regs { static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *fregs) { - return &fregs->regs; + struct pt_regs *regs = &fregs->regs; + + if (test_pt_regs_flag(regs, PIF_FTRACE_FULL_REGS)) + return regs; + return NULL; } static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *fregs, unsigned long ip) { - struct pt_regs *regs = arch_ftrace_get_regs(fregs); - - regs->psw.addr = ip; + fregs->regs.psw.addr = ip; } /* diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 4ffa8e7f0ed3..ddb70fb13fbc 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -15,11 +15,13 @@ #define PIF_EXECVE_PGSTE_RESTART 1 /* restart execve for PGSTE binaries */ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ #define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */ +#define PIF_FTRACE_FULL_REGS 4 /* all register contents valid (ftrace) */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_EXECVE_PGSTE_RESTART BIT(PIF_EXECVE_PGSTE_RESTART) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) #define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT) +#define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index d74e26b48604..5cb258cd9d29 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -17,22 +17,10 @@ #include <asm/ctl_reg.h> #include <asm/extable.h> #include <asm/facility.h> +#include <asm-generic/access_ok.h> void debug_user_asce(int exit); -static inline int __range_ok(unsigned long addr, unsigned long size) -{ - return 1; -} - -#define __access_ok(addr, size) \ -({ \ - __chk_user_ptr(addr); \ - __range_ok((unsigned long)(addr), (size)); \ -}) - -#define access_ok(addr, size) __access_ok(addr, size) - unsigned long __must_check raw_copy_from_user(void *to, const void __user *from, unsigned long n); @@ -282,8 +270,6 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo int copy_to_user_real(void __user *dest, void *src, unsigned long count); void *s390_kernel_write(void *dst, const void *src, size_t size); -#define HAVE_GET_KERNEL_NOFAULT - int __noreturn __put_kernel_bad(void); #define __put_kernel_asm(val, to, insn) \ diff --git a/arch/s390/include/uapi/asm/signal.h b/arch/s390/include/uapi/asm/signal.h index 0189f326aac5..e74d6ba1bd3b 100644 --- a/arch/s390/include/uapi/asm/signal.h +++ b/arch/s390/include/uapi/asm/signal.h @@ -108,7 +108,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 21d62d8b6b9a..89c0870d5679 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -159,9 +159,38 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) return 0; } +static struct ftrace_hotpatch_trampoline *ftrace_get_trampoline(struct dyn_ftrace *rec) +{ + struct ftrace_hotpatch_trampoline *trampoline; + struct ftrace_insn insn; + s64 disp; + u16 opc; + + if (copy_from_kernel_nofault(&insn, (void *)rec->ip, sizeof(insn))) + return ERR_PTR(-EFAULT); + disp = (s64)insn.disp * 2; + trampoline = (void *)(rec->ip + disp); + if (get_kernel_nofault(opc, &trampoline->brasl_opc)) + return ERR_PTR(-EFAULT); + if (opc != 0xc015) + return ERR_PTR(-EINVAL); + return trampoline; +} + int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { + struct ftrace_hotpatch_trampoline *trampoline; + u64 old; + + trampoline = ftrace_get_trampoline(rec); + if (IS_ERR(trampoline)) + return PTR_ERR(trampoline); + if (get_kernel_nofault(old, &trampoline->interceptor)) + return -EFAULT; + if (old != old_addr) + return -EINVAL; + s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr)); return 0; } @@ -188,6 +217,12 @@ static void brcl_enable(void *brcl) int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { + struct ftrace_hotpatch_trampoline *trampoline; + + trampoline = ftrace_get_trampoline(rec); + if (IS_ERR(trampoline)) + return PTR_ERR(trampoline); + s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr)); brcl_enable((void *)rec->ip); return 0; } @@ -291,7 +326,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, regs = ftrace_get_regs(fregs); p = get_kprobe((kprobe_opcode_t *)ip); - if (unlikely(!p) || kprobe_disabled(p)) + if (!regs || unlikely(!p) || kprobe_disabled(p)) goto out; if (kprobe_running()) { diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 39bcc0e39a10..a24177dcd12a 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -27,6 +27,7 @@ ENDPROC(ftrace_stub) #define STACK_PTREGS_GPRS (STACK_PTREGS + __PT_GPRS) #define STACK_PTREGS_PSW (STACK_PTREGS + __PT_PSW) #define STACK_PTREGS_ORIG_GPR2 (STACK_PTREGS + __PT_ORIG_GPR2) +#define STACK_PTREGS_FLAGS (STACK_PTREGS + __PT_FLAGS) #ifdef __PACK_STACK /* allocate just enough for r14, r15 and backchain */ #define TRACED_FUNC_FRAME_SIZE 24 @@ -57,6 +58,14 @@ ENDPROC(ftrace_stub) .if \allregs == 1 stg %r14,(STACK_PTREGS_PSW)(%r15) stosm (STACK_PTREGS_PSW)(%r15),0 +#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES + mvghi STACK_PTREGS_FLAGS(%r15),_PIF_FTRACE_FULL_REGS +#else + lghi %r14,_PIF_FTRACE_FULL_REGS + stg %r14,STACK_PTREGS_FLAGS(%r15) +#endif + .else + xc STACK_PTREGS_FLAGS(8,%r15),STACK_PTREGS_FLAGS(%r15) .endif lg %r14,(__SF_GPRS+8*8)(%r1) # restore original return address diff --git a/arch/s390/kernel/numa.c b/arch/s390/kernel/numa.c index 51c5a9f6e525..23ab9f02f278 100644 --- a/arch/s390/kernel/numa.c +++ b/arch/s390/kernel/numa.c @@ -33,10 +33,3 @@ void __init numa_setup(void) NODE_DATA(0)->node_spanned_pages = memblock_end_of_DRAM() >> PAGE_SHIFT; NODE_DATA(0)->node_id = 0; } - -static int __init numa_init_late(void) -{ - register_one_node(0); - return 0; -} -arch_initcall(numa_init_late); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index f2c25d113e7b..05327be3a982 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -800,6 +800,8 @@ static void __init check_initrd(void) static void __init reserve_kernel(void) { memblock_reserve(0, STARTUP_NORMAL_OFFSET); + memblock_reserve(OLDMEM_BASE, sizeof(unsigned long)); + memblock_reserve(OLDMEM_SIZE, sizeof(unsigned long)); memblock_reserve(__amode31_base, __eamode31 - __samode31); memblock_reserve(__pa(sclp_early_sccb), EXT_SCCB_READ_SCP); memblock_reserve(__pa(_stext), _end - _stext); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 577f1ead6a51..2296b1ff1e02 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4667,6 +4667,8 @@ static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu, return -EINVAL; if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block)) return -E2BIG; + if (!kvm_s390_pv_cpu_is_protected(vcpu)) + return -EINVAL; switch (mop->op) { case KVM_S390_MEMOP_SIDA_READ: diff --git a/arch/s390/lib/test_modules.c b/arch/s390/lib/test_modules.c index d056baa8fbb0..9894009fc1f2 100644 --- a/arch/s390/lib/test_modules.c +++ b/arch/s390/lib/test_modules.c @@ -5,9 +5,6 @@ #include "test_modules.h" -#define DECLARE_RETURN(i) int test_modules_return_ ## i(void) -REPEAT_10000(DECLARE_RETURN); - /* * Test that modules with many relocations are loaded properly. */ diff --git a/arch/s390/lib/test_modules.h b/arch/s390/lib/test_modules.h index 43b5e4b4af3e..6371fcf17684 100644 --- a/arch/s390/lib/test_modules.h +++ b/arch/s390/lib/test_modules.h @@ -47,4 +47,7 @@ __REPEAT_10000_1(f, 8); \ __REPEAT_10000_1(f, 9) +#define DECLARE_RETURN(i) int test_modules_return_ ## i(void) +REPEAT_10000(DECLARE_RETURN); + #endif diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c index a963c3d8ad0d..fb924a8041dc 100644 --- a/arch/s390/lib/xor.c +++ b/arch/s390/lib/xor.c @@ -11,7 +11,8 @@ #include <linux/raid/xor.h> #include <asm/xor.h> -static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { asm volatile( " larl 1,2f\n" @@ -32,8 +33,9 @@ static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) : "0", "1", "cc", "memory"); } -static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { asm volatile( " larl 1,2f\n" @@ -58,8 +60,10 @@ static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { asm volatile( " larl 1,2f\n" @@ -88,8 +92,11 @@ static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { asm volatile( " larl 1,2f\n" diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 2474a04ceac4..5f220e903e5a 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -7,6 +7,7 @@ config SUPERH select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) select ARCH_HAS_BINFMT_FLAT if !MMU + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_GIGANTIC_PAGE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_PTE_SPECIAL @@ -65,7 +66,6 @@ config SUPERH select PERF_EVENTS select PERF_USE_VMALLOC select RTC_LIB - select SET_FS select SPARSE_IRQ select TRACE_IRQFLAGS_SUPPORT help diff --git a/arch/sh/configs/ap325rxa_defconfig b/arch/sh/configs/ap325rxa_defconfig index 5193b3e099b9..4d83576b89c6 100644 --- a/arch/sh/configs/ap325rxa_defconfig +++ b/arch/sh/configs/ap325rxa_defconfig @@ -93,7 +93,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/sh/configs/ecovec24_defconfig b/arch/sh/configs/ecovec24_defconfig index 03cb916819fa..d90d29d44469 100644 --- a/arch/sh/configs/ecovec24_defconfig +++ b/arch/sh/configs/ecovec24_defconfig @@ -123,7 +123,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/sh/configs/landisk_defconfig b/arch/sh/configs/landisk_defconfig index e6c5ddf070c0..492a0a2e0e36 100644 --- a/arch/sh/configs/landisk_defconfig +++ b/arch/sh/configs/landisk_defconfig @@ -108,7 +108,6 @@ CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFS_V3=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_SMB_FS=m CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y diff --git a/arch/sh/configs/sdk7780_defconfig b/arch/sh/configs/sdk7780_defconfig index 6c719ab4332a..7d6d32359848 100644 --- a/arch/sh/configs/sdk7780_defconfig +++ b/arch/sh/configs/sdk7780_defconfig @@ -120,7 +120,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/sh/configs/se7724_defconfig b/arch/sh/configs/se7724_defconfig index a26f7f1841c7..d817df7cc4a7 100644 --- a/arch/sh/configs/se7724_defconfig +++ b/arch/sh/configs/se7724_defconfig @@ -122,7 +122,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/sh/configs/sh03_defconfig b/arch/sh/configs/sh03_defconfig index ff502683132e..9fcf68b22dba 100644 --- a/arch/sh/configs/sh03_defconfig +++ b/arch/sh/configs/sh03_defconfig @@ -75,7 +75,6 @@ CONFIG_NFS_V3=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_PARTITION_ADVANCED=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_737=m diff --git a/arch/sh/configs/sh7785lcr_32bit_defconfig b/arch/sh/configs/sh7785lcr_32bit_defconfig index 5c725c75fcef..7eb3c10f28ad 100644 --- a/arch/sh/configs/sh7785lcr_32bit_defconfig +++ b/arch/sh/configs/sh7785lcr_32bit_defconfig @@ -129,7 +129,6 @@ CONFIG_NFS_V3=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_NFSD=m -CONFIG_NFSD_V3=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y diff --git a/arch/sh/configs/titan_defconfig b/arch/sh/configs/titan_defconfig index cd5c58916c65..73a0d68b0de6 100644 --- a/arch/sh/configs/titan_defconfig +++ b/arch/sh/configs/titan_defconfig @@ -239,7 +239,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3=y CONFIG_ROOT_NFS=y CONFIG_NFSD=y -CONFIG_NFSD_V3=y CONFIG_SMB_FS=m CONFIG_CIFS=m CONFIG_PARTITION_ADVANCED=y diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 41be43e99cff..d0240decacca 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -406,6 +406,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return (unsigned long)pmd_val(pmd); } +#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) #define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) #ifdef CONFIG_X2TLB diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h index 3820d698846e..85a6c1c3c16e 100644 --- a/arch/sh/include/asm/processor.h +++ b/arch/sh/include/asm/processor.h @@ -3,7 +3,6 @@ #define __ASM_SH_PROCESSOR_H #include <asm/cpu-features.h> -#include <asm/segment.h> #include <asm/cache.h> #ifndef __ASSEMBLY__ diff --git a/arch/sh/include/asm/segment.h b/arch/sh/include/asm/segment.h deleted file mode 100644 index 02e54a3335d6..000000000000 --- a/arch/sh/include/asm/segment.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_SH_SEGMENT_H -#define __ASM_SH_SEGMENT_H - -#ifndef __ASSEMBLY__ - -typedef struct { - unsigned long seg; -} mm_segment_t; - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ -#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFUL) -#ifdef CONFIG_MMU -#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) -#else -#define USER_DS KERNEL_DS -#endif - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_SH_SEGMENT_H */ diff --git a/arch/sh/include/asm/thread_info.h b/arch/sh/include/asm/thread_info.h index 598d0184ffea..b119b859a0a3 100644 --- a/arch/sh/include/asm/thread_info.h +++ b/arch/sh/include/asm/thread_info.h @@ -30,7 +30,6 @@ struct thread_info { __u32 status; /* thread synchronous flags */ __u32 cpu; int preempt_count; /* 0 => preemptable, <0 => BUG */ - mm_segment_t addr_limit; /* thread address space */ unsigned long previous_sp; /* sp of previous stack in case of nested IRQ stacks */ __u8 supervisor_stack[0]; @@ -58,7 +57,6 @@ struct thread_info { .status = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } /* how to get the current stack pointer from C */ diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 8867bb04b00e..a79609eb14be 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -2,30 +2,8 @@ #ifndef __ASM_SH_UACCESS_H #define __ASM_SH_UACCESS_H -#include <asm/segment.h> #include <asm/extable.h> - -#define __addr_ok(addr) \ - ((unsigned long __force)(addr) < current_thread_info()->addr_limit.seg) - -/* - * __access_ok: Check if address with size is OK or not. - * - * Uhhuh, this needs 33-bit arithmetic. We have a carry.. - * - * sum := addr + size; carry? --> flag = true; - * if (sum >= addr_limit) flag = true; - */ -#define __access_ok(addr, size) ({ \ - unsigned long __ao_a = (addr), __ao_b = (size); \ - unsigned long __ao_end = __ao_a + __ao_b - !!__ao_b; \ - __ao_end >= __ao_a && __addr_ok(__ao_end); }) - -#define access_ok(addr, size) \ - (__chk_user_ptr(addr), \ - __access_ok((unsigned long __force)(addr), (size))) - -#define user_addr_max() (current_thread_info()->addr_limit.seg) +#include <asm-generic/access_ok.h> /* * Uh, these should become the main single-value transfer routines ... diff --git a/arch/sh/kernel/io_trapped.c b/arch/sh/kernel/io_trapped.c index 004ad0130b10..e803b14ef12e 100644 --- a/arch/sh/kernel/io_trapped.c +++ b/arch/sh/kernel/io_trapped.c @@ -270,7 +270,6 @@ static struct mem_access trapped_io_access = { int handle_trapped_io(struct pt_regs *regs, unsigned long address) { - mm_segment_t oldfs; insn_size_t instruction; int tmp; @@ -281,16 +280,12 @@ int handle_trapped_io(struct pt_regs *regs, unsigned long address) WARN_ON(user_mode(regs)); - oldfs = get_fs(); - set_fs(KERNEL_DS); - if (copy_from_user(&instruction, (void *)(regs->pc), - sizeof(instruction))) { - set_fs(oldfs); + if (copy_from_kernel_nofault(&instruction, (void *)(regs->pc), + sizeof(instruction))) { return 0; } tmp = handle_unaligned_access(instruction, regs, &trapped_io_access, 1, address); - set_fs(oldfs); return tmp == 0; } diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index 1c28e3cddb60..ca01286a0610 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -123,7 +123,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, #if defined(CONFIG_SH_FPU) childregs->sr |= SR_FD; #endif - ti->addr_limit = KERNEL_DS; ti->status &= ~TS_USEDFPU; p->thread.fpu_counter = 0; return 0; @@ -132,7 +131,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, if (usp) childregs->regs[15] = usp; - ti->addr_limit = USER_DS; if (clone_flags & CLONE_SETTLS) childregs->gbr = tls; diff --git a/arch/sh/kernel/topology.c b/arch/sh/kernel/topology.c index 76af6db9daa2..2d2a7509b565 100644 --- a/arch/sh/kernel/topology.c +++ b/arch/sh/kernel/topology.c @@ -46,11 +46,6 @@ static int __init topology_init(void) { int i, ret; -#ifdef CONFIG_NUMA - for_each_online_node(i) - register_one_node(i); -#endif - for_each_present_cpu(i) { struct cpu *c = &per_cpu(cpu_devices, i); diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c index b3c715bc254b..6cdda3a621a1 100644 --- a/arch/sh/kernel/traps_32.c +++ b/arch/sh/kernel/traps_32.c @@ -75,6 +75,23 @@ static struct mem_access user_mem_access = { copy_to_user, }; +static unsigned long copy_from_kernel_wrapper(void *dst, const void __user *src, + unsigned long cnt) +{ + return copy_from_kernel_nofault(dst, (const void __force *)src, cnt); +} + +static unsigned long copy_to_kernel_wrapper(void __user *dst, const void *src, + unsigned long cnt) +{ + return copy_to_kernel_nofault((void __force *)dst, src, cnt); +} + +static struct mem_access kernel_mem_access = { + copy_from_kernel_wrapper, + copy_to_kernel_wrapper, +}; + /* * handle an instruction that does an unaligned memory access by emulating the * desired behaviour @@ -473,7 +490,6 @@ asmlinkage void do_address_error(struct pt_regs *regs, unsigned long address) { unsigned long error_code = 0; - mm_segment_t oldfs; insn_size_t instruction; int tmp; @@ -489,13 +505,10 @@ asmlinkage void do_address_error(struct pt_regs *regs, local_irq_enable(); inc_unaligned_user_access(); - oldfs = force_uaccess_begin(); if (copy_from_user(&instruction, (insn_size_t __user *)(regs->pc & ~1), sizeof(instruction))) { - force_uaccess_end(oldfs); goto uspace_segv; } - force_uaccess_end(oldfs); /* shout about userspace fixups */ unaligned_fixups_notify(current, instruction, regs); @@ -518,11 +531,9 @@ fixup: goto uspace_segv; } - oldfs = force_uaccess_begin(); tmp = handle_unaligned_access(instruction, regs, &user_mem_access, 0, address); - force_uaccess_end(oldfs); if (tmp == 0) return; /* sorted */ @@ -538,21 +549,18 @@ uspace_segv: if (regs->pc & 1) die("unaligned program counter", regs, error_code); - set_fs(KERNEL_DS); - if (copy_from_user(&instruction, (void __user *)(regs->pc), + if (copy_from_kernel_nofault(&instruction, (void *)(regs->pc), sizeof(instruction))) { /* Argh. Fault on the instruction itself. This should never happen non-SMP */ - set_fs(oldfs); die("insn faulting in do_address_error", regs, 0); } unaligned_fixups_notify(current, instruction, regs); - handle_unaligned_access(instruction, regs, &user_mem_access, + handle_unaligned_access(instruction, regs, &kernel_mem_access, 0, address); - set_fs(oldfs); } } diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 1cab1b284f1a..9200bc04701c 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -46,7 +46,6 @@ config SPARC select LOCKDEP_SMALL if LOCKDEP select NEED_DMA_MAP_STATE select NEED_SG_DMA_LENGTH - select SET_FS select TRACE_IRQFLAGS_SUPPORT config SPARC32 @@ -62,6 +61,7 @@ config SPARC32 config SPARC64 def_bool 64BIT + select ALTERNATE_USER_ADDRESS_SPACE select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_KRETPROBES diff --git a/arch/sparc/include/asm/agp.h b/arch/sparc/include/asm/agp.h index efe0d6a12e5a..2d0ff84cee3f 100644 --- a/arch/sparc/include/asm/agp.h +++ b/arch/sparc/include/asm/agp.h @@ -4,9 +4,9 @@ /* dummy for now */ -#define map_page_into_agp(page) -#define unmap_page_from_agp(page) -#define flush_agp_cache() mb() +#define map_page_into_agp(page) do { } while (0) +#define unmap_page_from_agp(page) do { } while (0) +#define flush_agp_cache() mb() /* GATT allocation. Returns/accepts GATT kernel virtual address. */ #define alloc_gatt_pages(order) \ diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index ffccfe3b22ed..4866625da314 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -127,11 +127,16 @@ static inline int srmmu_device_memory(unsigned long x) return ((x & 0xF0000000) != 0); } +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return (pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4); +} + static inline struct page *pmd_page(pmd_t pmd) { if (srmmu_device_memory(pmd_val(pmd))) BUG(); - return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); + return pfn_to_page(pmd_pfn(pmd)); } static inline unsigned long __pmd_page(pmd_t pmd) diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index 647bf0ac7beb..b26c35336b51 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -32,10 +32,6 @@ struct fpq { }; #endif -typedef struct { - int seg; -} mm_segment_t; - /* The Sparc processor specific thread struct. */ struct thread_struct { struct pt_regs *kregs; @@ -50,11 +46,9 @@ struct thread_struct { unsigned long fsr; unsigned long fpqdepth; struct fpq fpqueue[16]; - mm_segment_t current_ds; }; #define INIT_THREAD { \ - .current_ds = KERNEL_DS, \ .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ } diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index ae851e8fce4c..89850dff6b03 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -47,10 +47,6 @@ #ifndef __ASSEMBLY__ -typedef struct { - unsigned char seg; -} mm_segment_t; - /* The Sparc processor specific thread struct. */ /* XXX This should die, everything can go into thread_info now. */ struct thread_struct { diff --git a/arch/sparc/include/asm/switch_to_64.h b/arch/sparc/include/asm/switch_to_64.h index b1d4e2e3210f..14f3c49bfdbc 100644 --- a/arch/sparc/include/asm/switch_to_64.h +++ b/arch/sparc/include/asm/switch_to_64.h @@ -20,10 +20,8 @@ do { \ */ #define switch_to(prev, next, last) \ do { save_and_clear_fpu(); \ - /* If you are tempted to conditionalize the following */ \ - /* so that ASI is only written if it changes, think again. */ \ __asm__ __volatile__("wr %%g0, %0, %%asi" \ - : : "r" (task_thread_info(next)->current_ds));\ + : : "r" (ASI_AIUS)); \ trap_block[current_thread_info()->cpu].thread = \ task_thread_info(next); \ __asm__ __volatile__( \ diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index 8047a9caab2f..1a44372e2bc0 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h @@ -46,7 +46,7 @@ struct thread_info { struct pt_regs *kregs; int preempt_count; /* 0 => preemptable, <0 => BUG */ __u8 new_child; - __u8 current_ds; + __u8 __pad; __u16 cpu; unsigned long *utraps; @@ -81,7 +81,6 @@ struct thread_info { #define TI_KREGS 0x00000028 #define TI_PRE_COUNT 0x00000030 #define TI_NEW_CHILD 0x00000034 -#define TI_CURRENT_DS 0x00000035 #define TI_CPU 0x00000036 #define TI_UTRAPS 0x00000038 #define TI_REG_WINDOW 0x00000040 @@ -116,7 +115,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .current_ds = ASI_P, \ .preempt_count = INIT_PREEMPT_COUNT, \ .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ } diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h index 390094200fc4..ee75f69e3fcd 100644 --- a/arch/sparc/include/asm/uaccess.h +++ b/arch/sparc/include/asm/uaccess.h @@ -10,9 +10,6 @@ #include <asm/uaccess_32.h> #endif -#define user_addr_max() \ - (uaccess_kernel() ? ~0UL : TASK_SIZE) - long strncpy_from_user(char *dest, const char __user *src, long count); #endif diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 4a12346bb69c..9fd6c53644b6 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -12,30 +12,7 @@ #include <linux/string.h> #include <asm/processor.h> - -/* Sparc is not segmented, however we need to be able to fool access_ok() - * when doing system calls from kernel mode legitimately. - * - * "For historical reasons, these macros are grossly misnamed." -Linus - */ - -#define KERNEL_DS ((mm_segment_t) { 0 }) -#define USER_DS ((mm_segment_t) { -1 }) - -#define get_fs() (current->thread.current_ds) -#define set_fs(val) ((current->thread.current_ds) = (val)) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -/* We have there a nice not-mapped page at PAGE_OFFSET - PAGE_SIZE, so that this test - * can be fairly lightweight. - * No one can read/write anything from userland in the kernel space by setting - * large size and address near to PAGE_OFFSET - a fault will break his intentions. - */ -#define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; }) -#define __kernel_ok (uaccess_kernel()) -#define __access_ok(addr, size) (__user_ok((addr) & get_fs().seg, (size))) -#define access_ok(addr, size) __access_ok((unsigned long)(addr), size) +#include <asm-generic/access_ok.h> /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right @@ -47,13 +24,13 @@ * and hide all the ugliness from the user. */ #define put_user(x, ptr) ({ \ - unsigned long __pu_addr = (unsigned long)(ptr); \ + void __user *__pu_addr = (ptr); \ __chk_user_ptr(ptr); \ __put_user_check((__typeof__(*(ptr)))(x), __pu_addr, sizeof(*(ptr))); \ }) #define get_user(x, ptr) ({ \ - unsigned long __gu_addr = (unsigned long)(ptr); \ + const void __user *__gu_addr = (ptr); \ __chk_user_ptr(ptr); \ __get_user_check((x), __gu_addr, sizeof(*(ptr)), __typeof__(*(ptr))); \ }) @@ -232,7 +209,7 @@ static inline unsigned long __clear_user(void __user *addr, unsigned long size) static inline unsigned long clear_user(void __user *addr, unsigned long n) { - if (n && __access_ok((unsigned long) addr, n)) + if (n && __access_ok(addr, n)) return __clear_user(addr, n); else return n; diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 30eb4c6414d1..94266a5c5b04 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -12,33 +12,15 @@ #include <asm/spitfire.h> #include <asm/processor.h> +#include <asm-generic/access_ok.h> /* * Sparc64 is segmented, though more like the M68K than the I386. * We use the secondary ASI to address user memory, which references a * completely different VM map, thus there is zero chance of the user * doing something queer and tricking us into poking kernel memory. - * - * What is left here is basically what is needed for the other parts of - * the kernel that expect to be able to manipulate, erum, "segments". - * Or perhaps more properly, permissions. - * - * "For historical reasons, these macros are grossly misnamed." -Linus */ -#define KERNEL_DS ((mm_segment_t) { ASI_P }) -#define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ - -#define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -#define set_fs(val) \ -do { \ - current_thread_info()->current_ds = (val).seg; \ - __asm__ __volatile__ ("wr %%g0, %0, %%asi" : : "r" ((val).seg)); \ -} while(0) - /* * Test whether a block of memory is a valid user space address. * Returns 0 if the range is valid, nonzero otherwise. @@ -61,16 +43,6 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ }) -static inline int __access_ok(const void __user * addr, unsigned long size) -{ - return 1; -} - -static inline int access_ok(const void __user * addr, unsigned long size) -{ - return 1; -} - void __retl_efault(void); /* Uh, these should become the main single-value transfer routines.. @@ -100,6 +72,42 @@ void __retl_efault(void); struct __large_struct { unsigned long buf[100]; }; #define __m(x) ((struct __large_struct *)(x)) +#define __put_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(dst); \ + type data = *(type *)src; \ + register int __pu_ret; \ + switch (sizeof(type)) { \ + case 1: __put_kernel_asm(data, b, addr, __pu_ret); break; \ + case 2: __put_kernel_asm(data, h, addr, __pu_ret); break; \ + case 4: __put_kernel_asm(data, w, addr, __pu_ret); break; \ + case 8: __put_kernel_asm(data, x, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + if (__pu_ret) \ + goto label; \ +} while (0) + +#define __put_kernel_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Put kernel asm, inline. */\n" \ + "1:\t" "st"#size " %1, [%2]\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=r" (ret) : "r" (x), "r" (__m(addr)), \ + "i" (-EFAULT)) + #define __put_user_nocheck(data, addr, size) ({ \ register int __pu_ret; \ switch (size) { \ @@ -134,6 +142,46 @@ __asm__ __volatile__( \ int __put_user_bad(void); +#define __get_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(src); \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (sizeof(type)) { \ + case 1: __get_kernel_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_kernel_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_kernel_asm(__gu_val, uw, addr, __gu_ret); break; \ + case 8: __get_kernel_asm(__gu_val, x, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + if (__gu_ret) \ + goto label; \ + *(type *)dst = (__force type) __gu_val; \ +} while (0) +#define __get_kernel_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Get kernel asm, inline. */\n" \ + "1:\t" "ld"#size " [%2], %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ + "i" (-EFAULT)) + #define __get_user_nocheck(data, addr, size, type) ({ \ register int __gu_ret; \ register unsigned long __gu_val; \ diff --git a/arch/sparc/include/asm/xor_32.h b/arch/sparc/include/asm/xor_32.h index 3e5af37e4b9c..0351813cf3af 100644 --- a/arch/sparc/include/asm/xor_32.h +++ b/arch/sparc/include/asm/xor_32.h @@ -13,7 +13,8 @@ */ static void -sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +sparc_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { int lines = bytes / (sizeof (long)) / 8; @@ -50,8 +51,9 @@ sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +sparc_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { int lines = bytes / (sizeof (long)) / 8; @@ -101,8 +103,10 @@ sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +sparc_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { int lines = bytes / (sizeof (long)) / 8; @@ -165,8 +169,11 @@ sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -sparc_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +sparc_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { int lines = bytes / (sizeof (long)) / 8; diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h index 16169f3edcd5..caaddea8ad79 100644 --- a/arch/sparc/include/asm/xor_64.h +++ b/arch/sparc/include/asm/xor_64.h @@ -12,13 +12,20 @@ #include <asm/spitfire.h> -void xor_vis_2(unsigned long, unsigned long *, unsigned long *); -void xor_vis_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -void xor_vis_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -void xor_vis_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); /* XXX Ugh, write cheetah versions... -DaveM */ @@ -30,13 +37,20 @@ static struct xor_block_template xor_block_VIS = { .do_5 = xor_vis_5, }; -void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); -void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); static struct xor_block_template xor_block_niagara = { .name = "Niagara", diff --git a/arch/sparc/include/uapi/asm/shmbuf.h b/arch/sparc/include/uapi/asm/shmbuf.h index a5d7d8d681c4..ed4f061c7a15 100644 --- a/arch/sparc/include/uapi/asm/shmbuf.h +++ b/arch/sparc/include/uapi/asm/shmbuf.h @@ -2,6 +2,9 @@ #ifndef _SPARC_SHMBUF_H #define _SPARC_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * The shmid64_ds structure for sparc architecture. * Note extra padding because this structure is passed back and forth @@ -25,7 +28,7 @@ struct shmid64_ds { unsigned long shm_ctime_high; unsigned long shm_ctime; /* last change time */ #endif - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ __kernel_pid_t shm_cpid; /* pid of creator */ __kernel_pid_t shm_lpid; /* pid of last operator */ unsigned long shm_nattch; /* no. of current attaches */ diff --git a/arch/sparc/include/uapi/asm/signal.h b/arch/sparc/include/uapi/asm/signal.h index 53758d53ac0e..b61382924725 100644 --- a/arch/sparc/include/uapi/asm/signal.h +++ b/arch/sparc/include/uapi/asm/signal.h @@ -151,6 +151,7 @@ struct sigstack { #include <asm-generic/signal-defs.h> +#include <asm/posix_types.h> #ifndef __KERNEL__ struct __new_sigaction { @@ -171,7 +172,7 @@ struct __old_sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index 2dc0bf9fe62e..88c0c14aaff0 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -300,7 +300,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, extern int nwindows; unsigned long psr; memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); - p->thread.current_ds = KERNEL_DS; ti->kpc = (((unsigned long) ret_from_kernel_thread) - 0x8); childregs->u_regs[UREG_G1] = sp; /* function */ childregs->u_regs[UREG_G2] = arg; @@ -311,7 +310,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, } memcpy(new_stack, (char *)regs - STACKFRAME_SZ, STACKFRAME_SZ + TRACEREG_SZ); childregs->u_regs[UREG_FP] = sp; - p->thread.current_ds = USER_DS; ti->kpc = (((unsigned long) ret_from_fork) - 0x8); ti->kpsr = current->thread.fork_kpsr | PSR_PIL; ti->kwim = current->thread.fork_kwim; diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index f5b2cac8669f..9a2ceb080ac9 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -106,18 +106,13 @@ static void show_regwindow32(struct pt_regs *regs) { struct reg_window32 __user *rw; struct reg_window32 r_w; - mm_segment_t old_fs; __asm__ __volatile__ ("flushw"); rw = compat_ptr((unsigned int)regs->u_regs[14]); - old_fs = get_fs(); - set_fs (USER_DS); if (copy_from_user (&r_w, rw, sizeof(r_w))) { - set_fs (old_fs); return; } - set_fs (old_fs); printk("l0: %08x l1: %08x l2: %08x l3: %08x " "l4: %08x l5: %08x l6: %08x l7: %08x\n", r_w.locals[0], r_w.locals[1], r_w.locals[2], r_w.locals[3], @@ -136,7 +131,6 @@ static void show_regwindow(struct pt_regs *regs) struct reg_window __user *rw; struct reg_window *rwk; struct reg_window r_w; - mm_segment_t old_fs; if ((regs->tstate & TSTATE_PRIV) || !(test_thread_flag(TIF_32BIT))) { __asm__ __volatile__ ("flushw"); @@ -145,14 +139,10 @@ static void show_regwindow(struct pt_regs *regs) rwk = (struct reg_window *) (regs->u_regs[14] + STACK_BIAS); if (!(regs->tstate & TSTATE_PRIV)) { - old_fs = get_fs(); - set_fs (USER_DS); if (copy_from_user (&r_w, rw, sizeof(r_w))) { - set_fs (old_fs); return; } rwk = &r_w; - set_fs (old_fs); } } else { show_regwindow32(regs); @@ -598,7 +588,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, memset(child_trap_frame, 0, child_stack_sz); __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (current_pt_regs()->tstate + 1) & TSTATE_CWP; - t->current_ds = ASI_P; t->kregs->u_regs[UREG_G1] = sp; /* function */ t->kregs->u_regs[UREG_G2] = arg; return 0; @@ -613,7 +602,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, t->kregs->u_regs[UREG_FP] = sp; __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (regs->tstate + 1) & TSTATE_CWP; - t->current_ds = ASI_AIUS; if (sp != regs->u_regs[UREG_FP]) { unsigned long csp; diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index ffab16369bea..74f80443b195 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -65,7 +65,7 @@ struct rt_signal_frame { */ static inline bool invalid_frame_pointer(void __user *fp, int fplen) { - if ((((unsigned long) fp) & 15) || !__access_ok((unsigned long)fp, fplen)) + if ((((unsigned long) fp) & 15) || !access_ok(fp, fplen)) return true; return false; diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 6d60d416f0dd..f19487e4cc71 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -244,22 +244,10 @@ static void __init check_mmu_stats(void) mmu_stats_supported = 1; } -static void register_nodes(void) -{ -#ifdef CONFIG_NUMA - int i; - - for (i = 0; i < MAX_NUMNODES; i++) - register_one_node(i); -#endif -} - static int __init topology_init(void) { int cpu, ret; - register_nodes(); - check_mmu_stats(); for_each_possible_cpu(cpu) { diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 21077821f427..5b4de4a89dec 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -2857,8 +2857,6 @@ void __init trap_init(void) TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) || TI_NEW_CHILD != offsetof(struct thread_info, new_child) || - TI_CURRENT_DS != offsetof(struct thread_info, - current_ds) || TI_KUNA_REGS != offsetof(struct thread_info, kern_una_regs) || TI_KUNA_INSN != offsetof(struct thread_info, diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S index 8e4d22a6ba0b..ee51c1230689 100644 --- a/arch/sparc/lib/NGmemcpy.S +++ b/arch/sparc/lib/NGmemcpy.S @@ -10,8 +10,7 @@ #include <asm/thread_info.h> #define GLOBAL_SPARE %g7 #define RESTORE_ASI(TMP) \ - ldub [%g6 + TI_CURRENT_DS], TMP; \ - wr TMP, 0x0, %asi; + wr %g0, ASI_AIUS, %asi #else #define GLOBAL_SPARE %g5 #define RESTORE_ASI(TMP) \ diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 0f49fada2093..d8e0e3c7038d 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -181,6 +181,7 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) { pte_t pte; + entry = pte_mkhuge(entry); pte = hugepage_shift_to_tte(entry, shift); #ifdef CONFIG_SPARC64 diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 1b23639e2fcd..8b1911591581 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -709,9 +709,10 @@ static void __init inherit_prom_mappings(void) void prom_world(int enter) { - if (!enter) - set_fs(get_fs()); - + /* + * No need to change the address space any more, just flush + * the register windows + */ __asm__ __volatile__("flushw"); } diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h index b9e20bbe2f75..167e236d9bb8 100644 --- a/arch/um/include/asm/pgtable.h +++ b/arch/um/include/asm/pgtable.h @@ -109,6 +109,7 @@ extern unsigned long end_iomem; #define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE) #define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE) +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK) #define pte_page(x) pfn_to_page(pte_pfn(x)) diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 17d18cfd82a5..7d9d60e41e4e 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -25,7 +25,7 @@ extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); extern unsigned long __clear_user(void __user *mem, unsigned long len); -static inline int __access_ok(unsigned long addr, unsigned long size); +static inline int __access_ok(const void __user *ptr, unsigned long size); /* Teach asm-generic/uaccess.h that we have C functions for these. */ #define __access_ok __access_ok @@ -36,16 +36,15 @@ static inline int __access_ok(unsigned long addr, unsigned long size); #include <asm-generic/uaccess.h> -static inline int __access_ok(unsigned long addr, unsigned long size) +static inline int __access_ok(const void __user *ptr, unsigned long size) { + unsigned long addr = (unsigned long)ptr; return __addr_range_nowrap(addr, size) && (__under_task_size(addr, size) || __access_ok_vsyscall(addr, size)); } /* no pagefaults for kernel addresses in um */ -#define HAVE_GET_KERNEL_NOFAULT 1 - #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ *((type *)dst) = get_unaligned((type *)(src)); \ diff --git a/arch/um/os-Linux/execvp.c b/arch/um/os-Linux/execvp.c index 84a0777c2a45..c09a5fd5e225 100644 --- a/arch/um/os-Linux/execvp.c +++ b/arch/um/os-Linux/execvp.c @@ -93,6 +93,7 @@ int execvp_noalloc(char *buf, const char *file, char *const argv[]) up finding no executable we can use, we want to diagnose that we did find one but were denied access. */ got_eacces = 1; + break; case ENOENT: case ESTALE: case ENOTDIR: diff --git a/arch/x86/Kbuild b/arch/x86/Kbuild index f384cb1a4f7a..5a83da703e87 100644 --- a/arch/x86/Kbuild +++ b/arch/x86/Kbuild @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ARCH_HAS_CC_PLATFORM) += coco/ + obj-y += entry/ obj-$(CONFIG_PERF_EVENTS) += events/ diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9f5bd41bf660..61e511c51890 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -69,6 +69,7 @@ config X86 select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CACHE_LINE_SIZE + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE select ARCH_HAS_DEVMEM_IS_ALLOWED @@ -118,8 +119,10 @@ config X86 select ARCH_WANT_DEFAULT_BPF_JIT if X86_64 select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANTS_NO_INSTR + select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE select ARCH_WANT_LD_ORPHAN_WARN + select ARCH_WANTS_RT_DELAYED_SIGNALS select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select BUILDTIME_TABLE_SORT @@ -246,7 +249,7 @@ config X86 select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION - select HAVE_PREEMPT_DYNAMIC + select HAVE_PREEMPT_DYNAMIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK @@ -347,9 +350,6 @@ config ARCH_NR_GPIO config ARCH_SUSPEND_POSSIBLE def_bool y -config ARCH_WANT_GENERAL_HUGETLB - def_bool y - config AUDIT_ARCH def_bool y if X86_64 @@ -1275,23 +1275,6 @@ config TOSHIBA Say Y if you intend to run this kernel on a Toshiba portable. Say N otherwise. -config I8K - tristate "Dell i8k legacy laptop support" - depends on HWMON - depends on PROC_FS - select SENSORS_DELL_SMM - help - This option enables legacy /proc/i8k userspace interface in hwmon - dell-smm-hwmon driver. Character file /proc/i8k reports bios version, - temperature and allows controlling fan speeds of Dell laptops via - System Management Mode. For old Dell laptops (like Dell Inspiron 8000) - it reports also power and hotkey status. For fan speed control is - needed userspace package i8kutils. - - Say Y if you intend to run this kernel on old Dell laptops or want to - use userspace package i8kutils. - Say N otherwise. - config X86_REBOOTFIXUPS bool "Enable X86 board specific fixups for reboot" depends on X86_32 @@ -1638,7 +1621,7 @@ config ARCH_SPARSEMEM_DEFAULT config ARCH_SELECT_MEMORY_MODEL def_bool y - depends on ARCH_SPARSEMEM_ENABLE + depends on ARCH_SPARSEMEM_ENABLE && ARCH_FLATMEM_ENABLE config ARCH_MEMORY_PROBE bool "Enable sysfs memory/probe interface" diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index 659fad53ca82..3b354eb9516d 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -152,14 +152,13 @@ SYM_FUNC_END(startup_32) #ifdef CONFIG_EFI_STUB SYM_FUNC_START(efi32_stub_entry) -SYM_FUNC_START_ALIAS(efi_stub_entry) add $0x4, %esp movl 8(%esp), %esi /* save boot_params pointer */ call efi_main /* efi_main returns the possibly relocated address of startup_32 */ jmp *%eax SYM_FUNC_END(efi32_stub_entry) -SYM_FUNC_END_ALIAS(efi_stub_entry) +SYM_FUNC_ALIAS(efi_stub_entry, efi32_stub_entry) #endif .text diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index fd9441f40457..dea95301196b 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -535,7 +535,6 @@ SYM_CODE_END(startup_64) #ifdef CONFIG_EFI_STUB .org 0x390 SYM_FUNC_START(efi64_stub_entry) -SYM_FUNC_START_ALIAS(efi_stub_entry) and $~0xf, %rsp /* realign the stack */ movq %rdx, %rbx /* save boot_params pointer */ call efi_main @@ -543,7 +542,7 @@ SYM_FUNC_START_ALIAS(efi_stub_entry) leaq rva(startup_64)(%rax), %rax jmp *%rax SYM_FUNC_END(efi64_stub_entry) -SYM_FUNC_END_ALIAS(efi_stub_entry) +SYM_FUNC_ALIAS(efi_stub_entry, efi64_stub_entry) #endif .text diff --git a/arch/x86/coco/Makefile b/arch/x86/coco/Makefile new file mode 100644 index 000000000000..c1ead00017a7 --- /dev/null +++ b/arch/x86/coco/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +CFLAGS_REMOVE_core.o = -pg +KASAN_SANITIZE_core.o := n +CFLAGS_core.o += -fno-stack-protector + +obj-y += core.o diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/coco/core.c index 6a6ffcd978f6..fc1365dd927e 100644 --- a/arch/x86/kernel/cc_platform.c +++ b/arch/x86/coco/core.c @@ -9,18 +9,16 @@ #include <linux/export.h> #include <linux/cc_platform.h> -#include <linux/mem_encrypt.h> -#include <asm/mshyperv.h> +#include <asm/coco.h> #include <asm/processor.h> -static bool __maybe_unused intel_cc_platform_has(enum cc_attr attr) +static enum cc_vendor vendor __ro_after_init; +static u64 cc_mask __ro_after_init; + +static bool intel_cc_platform_has(enum cc_attr attr) { -#ifdef CONFIG_INTEL_TDX_GUEST - return false; -#else return false; -#endif } /* @@ -74,12 +72,46 @@ static bool hyperv_cc_platform_has(enum cc_attr attr) bool cc_platform_has(enum cc_attr attr) { - if (sme_me_mask) + switch (vendor) { + case CC_VENDOR_AMD: return amd_cc_platform_has(attr); - - if (hv_is_isolation_supported()) + case CC_VENDOR_INTEL: + return intel_cc_platform_has(attr); + case CC_VENDOR_HYPERV: return hyperv_cc_platform_has(attr); - - return false; + default: + return false; + } } EXPORT_SYMBOL_GPL(cc_platform_has); + +u64 cc_mkenc(u64 val) +{ + switch (vendor) { + case CC_VENDOR_AMD: + return val | cc_mask; + default: + return val; + } +} + +u64 cc_mkdec(u64 val) +{ + switch (vendor) { + case CC_VENDOR_AMD: + return val & ~cc_mask; + default: + return val; + } +} +EXPORT_SYMBOL_GPL(cc_mkdec); + +__init void cc_set_vendor(enum cc_vendor v) +{ + vendor = v; +} + +__init void cc_set_mask(u64 mask) +{ + cc_mask = mask; +} diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index c3af959648e6..2831685adf6f 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -90,6 +90,9 @@ nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o +obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o +sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o + obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S index c799838242a6..43852ba6e19c 100644 --- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S +++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S @@ -1,65 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */ /* - * Implement AES CTR mode by8 optimization with AVX instructions. (x86_64) - * - * This is AES128/192/256 CTR mode optimization implementation. It requires - * the support of Intel(R) AESNI and AVX instructions. - * - * This work was inspired by the AES CTR mode optimization published - * in Intel Optimized IPSEC Cryptograhpic library. - * Additional information on it can be found at: - * http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972 - * - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY + * AES CTR mode by8 optimization with AVX instructions. (x86_64) * * Copyright(c) 2014 Intel Corporation. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * * Contact Information: * James Guilford <james.guilford@intel.com> * Sean Gulley <sean.m.gulley@intel.com> * Chandramouli Narayanan <mouli@linux.intel.com> + */ +/* + * This is AES128/192/256 CTR mode optimization implementation. It requires + * the support of Intel(R) AESNI and AVX instructions. * - * BSD LICENSE - * - * Copyright(c) 2014 Intel Corporation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * + * This work was inspired by the AES CTR mode optimization published + * in Intel Optimized IPSEC Cryptographic library. + * Additional information on it can be found at: + * https://github.com/intel/intel-ipsec-mb */ #include <linux/linkage.h> diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 363699dd7220..837c1e0aa021 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -1751,8 +1751,6 @@ SYM_FUNC_END(aesni_gcm_finalize) #endif - -SYM_FUNC_START_LOCAL_ALIAS(_key_expansion_128) SYM_FUNC_START_LOCAL(_key_expansion_256a) pshufd $0b11111111, %xmm1, %xmm1 shufps $0b00010000, %xmm0, %xmm4 @@ -1764,7 +1762,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_256a) add $0x10, TKEYP RET SYM_FUNC_END(_key_expansion_256a) -SYM_FUNC_END_ALIAS(_key_expansion_128) +SYM_FUNC_ALIAS_LOCAL(_key_expansion_128, _key_expansion_256a) SYM_FUNC_START_LOCAL(_key_expansion_192a) pshufd $0b01010101, %xmm1, %xmm1 diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index a880e0b1c255..fda6066437aa 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -32,24 +32,12 @@ static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) __blowfish_enc_blk(ctx, dst, src, false); } -static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst, - const u8 *src) -{ - __blowfish_enc_blk(ctx, dst, src, true); -} - static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, const u8 *src) { __blowfish_enc_blk_4way(ctx, dst, src, false); } -static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst, - const u8 *src) -{ - __blowfish_enc_blk_4way(ctx, dst, src, true); -} - static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src); diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index 787c234d2469..abb8b1fe123b 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -45,14 +45,6 @@ static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); } -static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, - const u8 *src) -{ - u32 *enc_ctx = ctx->enc.expkey; - - des3_ede_x86_64_crypt_blk_3way(enc_ctx, dst, src); -} - static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, const u8 *src) { diff --git a/arch/x86/crypto/sm3-avx-asm_64.S b/arch/x86/crypto/sm3-avx-asm_64.S new file mode 100644 index 000000000000..71e6aae23e17 --- /dev/null +++ b/arch/x86/crypto/sm3-avx-asm_64.S @@ -0,0 +1,517 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 AVX accelerated transform. + * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 + * + * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +/* Based on SM3 AES/BMI2 accelerated work by libgcrypt at: + * https://gnupg.org/software/libgcrypt/index.html + */ + +#include <linux/linkage.h> +#include <asm/frame.h> + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 +#define state_h5 20 +#define state_h6 24 +#define state_h7 28 + +/* Constants */ + +/* Round constant macros */ + +#define K0 2043430169 /* 0x79cc4519 */ +#define K1 -208106958 /* 0xf3988a32 */ +#define K2 -416213915 /* 0xe7311465 */ +#define K3 -832427829 /* 0xce6228cb */ +#define K4 -1664855657 /* 0x9cc45197 */ +#define K5 965255983 /* 0x3988a32f */ +#define K6 1930511966 /* 0x7311465e */ +#define K7 -433943364 /* 0xe6228cbc */ +#define K8 -867886727 /* 0xcc451979 */ +#define K9 -1735773453 /* 0x988a32f3 */ +#define K10 823420391 /* 0x311465e7 */ +#define K11 1646840782 /* 0x6228cbce */ +#define K12 -1001285732 /* 0xc451979c */ +#define K13 -2002571463 /* 0x88a32f39 */ +#define K14 289824371 /* 0x11465e73 */ +#define K15 579648742 /* 0x228cbce6 */ +#define K16 -1651869049 /* 0x9d8a7a87 */ +#define K17 991229199 /* 0x3b14f50f */ +#define K18 1982458398 /* 0x7629ea1e */ +#define K19 -330050500 /* 0xec53d43c */ +#define K20 -660100999 /* 0xd8a7a879 */ +#define K21 -1320201997 /* 0xb14f50f3 */ +#define K22 1654563303 /* 0x629ea1e7 */ +#define K23 -985840690 /* 0xc53d43ce */ +#define K24 -1971681379 /* 0x8a7a879d */ +#define K25 351604539 /* 0x14f50f3b */ +#define K26 703209078 /* 0x29ea1e76 */ +#define K27 1406418156 /* 0x53d43cec */ +#define K28 -1482130984 /* 0xa7a879d8 */ +#define K29 1330705329 /* 0x4f50f3b1 */ +#define K30 -1633556638 /* 0x9ea1e762 */ +#define K31 1027854021 /* 0x3d43cec5 */ +#define K32 2055708042 /* 0x7a879d8a */ +#define K33 -183551212 /* 0xf50f3b14 */ +#define K34 -367102423 /* 0xea1e7629 */ +#define K35 -734204845 /* 0xd43cec53 */ +#define K36 -1468409689 /* 0xa879d8a7 */ +#define K37 1358147919 /* 0x50f3b14f */ +#define K38 -1578671458 /* 0xa1e7629e */ +#define K39 1137624381 /* 0x43cec53d */ +#define K40 -2019718534 /* 0x879d8a7a */ +#define K41 255530229 /* 0x0f3b14f5 */ +#define K42 511060458 /* 0x1e7629ea */ +#define K43 1022120916 /* 0x3cec53d4 */ +#define K44 2044241832 /* 0x79d8a7a8 */ +#define K45 -206483632 /* 0xf3b14f50 */ +#define K46 -412967263 /* 0xe7629ea1 */ +#define K47 -825934525 /* 0xcec53d43 */ +#define K48 -1651869049 /* 0x9d8a7a87 */ +#define K49 991229199 /* 0x3b14f50f */ +#define K50 1982458398 /* 0x7629ea1e */ +#define K51 -330050500 /* 0xec53d43c */ +#define K52 -660100999 /* 0xd8a7a879 */ +#define K53 -1320201997 /* 0xb14f50f3 */ +#define K54 1654563303 /* 0x629ea1e7 */ +#define K55 -985840690 /* 0xc53d43ce */ +#define K56 -1971681379 /* 0x8a7a879d */ +#define K57 351604539 /* 0x14f50f3b */ +#define K58 703209078 /* 0x29ea1e76 */ +#define K59 1406418156 /* 0x53d43cec */ +#define K60 -1482130984 /* 0xa7a879d8 */ +#define K61 1330705329 /* 0x4f50f3b1 */ +#define K62 -1633556638 /* 0x9ea1e762 */ +#define K63 1027854021 /* 0x3d43cec5 */ + +/* Register macros */ + +#define RSTATE %rdi +#define RDATA %rsi +#define RNBLKS %rdx + +#define t0 %eax +#define t1 %ebx +#define t2 %ecx + +#define a %r8d +#define b %r9d +#define c %r10d +#define d %r11d +#define e %r12d +#define f %r13d +#define g %r14d +#define h %r15d + +#define W0 %xmm0 +#define W1 %xmm1 +#define W2 %xmm2 +#define W3 %xmm3 +#define W4 %xmm4 +#define W5 %xmm5 + +#define XTMP0 %xmm6 +#define XTMP1 %xmm7 +#define XTMP2 %xmm8 +#define XTMP3 %xmm9 +#define XTMP4 %xmm10 +#define XTMP5 %xmm11 +#define XTMP6 %xmm12 + +#define BSWAP_REG %xmm15 + +/* Stack structure */ + +#define STACK_W_SIZE (32 * 2 * 3) +#define STACK_REG_SAVE_SIZE (64) + +#define STACK_W (0) +#define STACK_REG_SAVE (STACK_W + STACK_W_SIZE) +#define STACK_SIZE (STACK_REG_SAVE + STACK_REG_SAVE_SIZE) + +/* Instruction helpers. */ + +#define roll2(v, reg) \ + roll $(v), reg; + +#define roll3mov(v, src, dst) \ + movl src, dst; \ + roll $(v), dst; + +#define roll3(v, src, dst) \ + rorxl $(32-(v)), src, dst; + +#define addl2(a, out) \ + leal (a, out), out; + +/* Round function macros. */ + +#define GG1(x, y, z, o, t) \ + movl x, o; \ + xorl y, o; \ + xorl z, o; + +#define FF1(x, y, z, o, t) GG1(x, y, z, o, t) + +#define GG2(x, y, z, o, t) \ + andnl z, x, o; \ + movl y, t; \ + andl x, t; \ + addl2(t, o); + +#define FF2(x, y, z, o, t) \ + movl y, o; \ + xorl x, o; \ + movl y, t; \ + andl x, t; \ + andl z, o; \ + xorl t, o; + +#define R(i, a, b, c, d, e, f, g, h, round, widx, wtype) \ + /* rol(a, 12) => t0 */ \ + roll3mov(12, a, t0); /* rorxl here would reduce perf by 6% on zen3 */ \ + /* rol (t0 + e + t), 7) => t1 */ \ + leal K##round(t0, e, 1), t1; \ + roll2(7, t1); \ + /* h + w1 => h */ \ + addl wtype##_W1_ADDR(round, widx), h; \ + /* h + t1 => h */ \ + addl2(t1, h); \ + /* t1 ^ t0 => t0 */ \ + xorl t1, t0; \ + /* w1w2 + d => d */ \ + addl wtype##_W1W2_ADDR(round, widx), d; \ + /* FF##i(a,b,c) => t1 */ \ + FF##i(a, b, c, t1, t2); \ + /* d + t1 => d */ \ + addl2(t1, d); \ + /* GG#i(e,f,g) => t2 */ \ + GG##i(e, f, g, t2, t1); \ + /* h + t2 => h */ \ + addl2(t2, h); \ + /* rol (f, 19) => f */ \ + roll2(19, f); \ + /* d + t0 => d */ \ + addl2(t0, d); \ + /* rol (b, 9) => b */ \ + roll2(9, b); \ + /* P0(h) => h */ \ + roll3(9, h, t2); \ + roll3(17, h, t1); \ + xorl t2, h; \ + xorl t1, h; + +#define R1(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(1, a, b, c, d, e, f, g, h, round, widx, wtype) + +#define R2(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(2, a, b, c, d, e, f, g, h, round, widx, wtype) + +/* Input expansion macros. */ + +/* Byte-swapped input address. */ +#define IW_W_ADDR(round, widx, offs) \ + (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Expanded input address. */ +#define XW_W_ADDR(round, widx, offs) \ + (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Rounds 1-12, byte-swapped input block addresses. */ +#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 0) +#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 32) + +/* Rounds 1-12, expanded input block addresses. */ +#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) +#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 32) + +/* Input block loading. */ +#define LOAD_W_XMM_1() \ + vmovdqu 0*16(RDATA), XTMP0; /* XTMP0: w3, w2, w1, w0 */ \ + vmovdqu 1*16(RDATA), XTMP1; /* XTMP1: w7, w6, w5, w4 */ \ + vmovdqu 2*16(RDATA), XTMP2; /* XTMP2: w11, w10, w9, w8 */ \ + vmovdqu 3*16(RDATA), XTMP3; /* XTMP3: w15, w14, w13, w12 */ \ + vpshufb BSWAP_REG, XTMP0, XTMP0; \ + vpshufb BSWAP_REG, XTMP1, XTMP1; \ + vpshufb BSWAP_REG, XTMP2, XTMP2; \ + vpshufb BSWAP_REG, XTMP3, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP4; \ + vpxor XTMP1, XTMP2, XTMP5; \ + vpxor XTMP2, XTMP3, XTMP6; \ + leaq 64(RDATA), RDATA; \ + vmovdqa XTMP0, IW_W1_ADDR(0, 0); \ + vmovdqa XTMP4, IW_W1W2_ADDR(0, 0); \ + vmovdqa XTMP1, IW_W1_ADDR(4, 0); \ + vmovdqa XTMP5, IW_W1W2_ADDR(4, 0); + +#define LOAD_W_XMM_2() \ + vmovdqa XTMP2, IW_W1_ADDR(8, 0); \ + vmovdqa XTMP6, IW_W1W2_ADDR(8, 0); + +#define LOAD_W_XMM_3() \ + vpshufd $0b00000000, XTMP0, W0; /* W0: xx, w0, xx, xx */ \ + vpshufd $0b11111001, XTMP0, W1; /* W1: xx, w3, w2, w1 */ \ + vmovdqa XTMP1, W2; /* W2: xx, w6, w5, w4 */ \ + vpalignr $12, XTMP1, XTMP2, W3; /* W3: xx, w9, w8, w7 */ \ + vpalignr $8, XTMP2, XTMP3, W4; /* W4: xx, w12, w11, w10 */ \ + vpshufd $0b11111001, XTMP3, W5; /* W5: xx, w15, w14, w13 */ + +/* Message scheduling. Note: 3 words per XMM register. */ +#define SCHED_W_0(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 16]) => XTMP0 */ \ + vpshufd $0b10111111, w0, XTMP0; \ + vpalignr $12, XTMP0, w1, XTMP0; /* XTMP0: xx, w2, w1, w0 */ \ + /* Load (w[i - 13]) => XTMP1 */ \ + vpshufd $0b10111111, w1, XTMP1; \ + vpalignr $12, XTMP1, w2, XTMP1; \ + /* w[i - 9] == w3 */ \ + /* XMM3 ^ XTMP0 => XTMP0 */ \ + vpxor w3, XTMP0, XTMP0; + +#define SCHED_W_1(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 3] == w5 */ \ + /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ + vpslld $15, w5, XTMP2; \ + vpsrld $(32-15), w5, XTMP3; \ + vpxor XTMP2, XTMP3, XTMP3; \ + vpxor XTMP3, XTMP0, XTMP0; \ + /* rol(XTMP1, 7) => XTMP1 */ \ + vpslld $7, XTMP1, XTMP5; \ + vpsrld $(32-7), XTMP1, XTMP1; \ + vpxor XTMP5, XTMP1, XTMP1; \ + /* XMM4 ^ XTMP1 => XTMP1 */ \ + vpxor w4, XTMP1, XTMP1; \ + /* w[i - 6] == XMM4 */ \ + /* P1(XTMP0) ^ XTMP1 => XMM0 */ \ + vpslld $15, XTMP0, XTMP5; \ + vpsrld $(32-15), XTMP0, XTMP6; \ + vpslld $23, XTMP0, XTMP2; \ + vpsrld $(32-23), XTMP0, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP1; \ + vpxor XTMP6, XTMP5, XTMP5; \ + vpxor XTMP3, XTMP2, XTMP2; \ + vpxor XTMP2, XTMP5, XTMP5; \ + vpxor XTMP5, XTMP1, w0; + +#define SCHED_W_2(round, w0, w1, w2, w3, w4, w5) \ + /* W1 in XMM12 */ \ + vpshufd $0b10111111, w4, XTMP4; \ + vpalignr $12, XTMP4, w5, XTMP4; \ + vmovdqa XTMP4, XW_W1_ADDR((round), 0); \ + /* W1 ^ W2 => XTMP1 */ \ + vpxor w0, XTMP4, XTMP1; \ + vmovdqa XTMP1, XW_W1W2_ADDR((round), 0); + + +.section .rodata.cst16, "aM", @progbits, 16 +.align 16 + +.Lbe32mask: + .long 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f + +.text + +/* + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. + * + * void sm3_transform_avx(struct sm3_state *state, + * const u8 *data, int nblocks); + */ +.align 16 +SYM_FUNC_START(sm3_transform_avx) + /* input: + * %rdi: ctx, CTX + * %rsi: data (64*nblks bytes) + * %rdx: nblocks + */ + vzeroupper; + + pushq %rbp; + movq %rsp, %rbp; + + movq %rdx, RNBLKS; + + subq $STACK_SIZE, %rsp; + andq $(~63), %rsp; + + movq %rbx, (STACK_REG_SAVE + 0 * 8)(%rsp); + movq %r15, (STACK_REG_SAVE + 1 * 8)(%rsp); + movq %r14, (STACK_REG_SAVE + 2 * 8)(%rsp); + movq %r13, (STACK_REG_SAVE + 3 * 8)(%rsp); + movq %r12, (STACK_REG_SAVE + 4 * 8)(%rsp); + + vmovdqa .Lbe32mask (%rip), BSWAP_REG; + + /* Get the values of the chaining variables. */ + movl state_h0(RSTATE), a; + movl state_h1(RSTATE), b; + movl state_h2(RSTATE), c; + movl state_h3(RSTATE), d; + movl state_h4(RSTATE), e; + movl state_h5(RSTATE), f; + movl state_h6(RSTATE), g; + movl state_h7(RSTATE), h; + +.align 16 +.Loop: + /* Load data part1. */ + LOAD_W_XMM_1(); + + leaq -1(RNBLKS), RNBLKS; + + /* Transform 0-3 + Load data part2. */ + R1(a, b, c, d, e, f, g, h, 0, 0, IW); LOAD_W_XMM_2(); + R1(d, a, b, c, h, e, f, g, 1, 1, IW); + R1(c, d, a, b, g, h, e, f, 2, 2, IW); + R1(b, c, d, a, f, g, h, e, 3, 3, IW); LOAD_W_XMM_3(); + + /* Transform 4-7 + Precalc 12-14. */ + R1(a, b, c, d, e, f, g, h, 4, 0, IW); + R1(d, a, b, c, h, e, f, g, 5, 1, IW); + R1(c, d, a, b, g, h, e, f, 6, 2, IW); SCHED_W_0(12, W0, W1, W2, W3, W4, W5); + R1(b, c, d, a, f, g, h, e, 7, 3, IW); SCHED_W_1(12, W0, W1, W2, W3, W4, W5); + + /* Transform 8-11 + Precalc 12-17. */ + R1(a, b, c, d, e, f, g, h, 8, 0, IW); SCHED_W_2(12, W0, W1, W2, W3, W4, W5); + R1(d, a, b, c, h, e, f, g, 9, 1, IW); SCHED_W_0(15, W1, W2, W3, W4, W5, W0); + R1(c, d, a, b, g, h, e, f, 10, 2, IW); SCHED_W_1(15, W1, W2, W3, W4, W5, W0); + R1(b, c, d, a, f, g, h, e, 11, 3, IW); SCHED_W_2(15, W1, W2, W3, W4, W5, W0); + + /* Transform 12-14 + Precalc 18-20 */ + R1(a, b, c, d, e, f, g, h, 12, 0, XW); SCHED_W_0(18, W2, W3, W4, W5, W0, W1); + R1(d, a, b, c, h, e, f, g, 13, 1, XW); SCHED_W_1(18, W2, W3, W4, W5, W0, W1); + R1(c, d, a, b, g, h, e, f, 14, 2, XW); SCHED_W_2(18, W2, W3, W4, W5, W0, W1); + + /* Transform 15-17 + Precalc 21-23 */ + R1(b, c, d, a, f, g, h, e, 15, 0, XW); SCHED_W_0(21, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 16, 1, XW); SCHED_W_1(21, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 17, 2, XW); SCHED_W_2(21, W3, W4, W5, W0, W1, W2); + + /* Transform 18-20 + Precalc 24-26 */ + R2(c, d, a, b, g, h, e, f, 18, 0, XW); SCHED_W_0(24, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 19, 1, XW); SCHED_W_1(24, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 20, 2, XW); SCHED_W_2(24, W4, W5, W0, W1, W2, W3); + + /* Transform 21-23 + Precalc 27-29 */ + R2(d, a, b, c, h, e, f, g, 21, 0, XW); SCHED_W_0(27, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 22, 1, XW); SCHED_W_1(27, W5, W0, W1, W2, W3, W4); + R2(b, c, d, a, f, g, h, e, 23, 2, XW); SCHED_W_2(27, W5, W0, W1, W2, W3, W4); + + /* Transform 24-26 + Precalc 30-32 */ + R2(a, b, c, d, e, f, g, h, 24, 0, XW); SCHED_W_0(30, W0, W1, W2, W3, W4, W5); + R2(d, a, b, c, h, e, f, g, 25, 1, XW); SCHED_W_1(30, W0, W1, W2, W3, W4, W5); + R2(c, d, a, b, g, h, e, f, 26, 2, XW); SCHED_W_2(30, W0, W1, W2, W3, W4, W5); + + /* Transform 27-29 + Precalc 33-35 */ + R2(b, c, d, a, f, g, h, e, 27, 0, XW); SCHED_W_0(33, W1, W2, W3, W4, W5, W0); + R2(a, b, c, d, e, f, g, h, 28, 1, XW); SCHED_W_1(33, W1, W2, W3, W4, W5, W0); + R2(d, a, b, c, h, e, f, g, 29, 2, XW); SCHED_W_2(33, W1, W2, W3, W4, W5, W0); + + /* Transform 30-32 + Precalc 36-38 */ + R2(c, d, a, b, g, h, e, f, 30, 0, XW); SCHED_W_0(36, W2, W3, W4, W5, W0, W1); + R2(b, c, d, a, f, g, h, e, 31, 1, XW); SCHED_W_1(36, W2, W3, W4, W5, W0, W1); + R2(a, b, c, d, e, f, g, h, 32, 2, XW); SCHED_W_2(36, W2, W3, W4, W5, W0, W1); + + /* Transform 33-35 + Precalc 39-41 */ + R2(d, a, b, c, h, e, f, g, 33, 0, XW); SCHED_W_0(39, W3, W4, W5, W0, W1, W2); + R2(c, d, a, b, g, h, e, f, 34, 1, XW); SCHED_W_1(39, W3, W4, W5, W0, W1, W2); + R2(b, c, d, a, f, g, h, e, 35, 2, XW); SCHED_W_2(39, W3, W4, W5, W0, W1, W2); + + /* Transform 36-38 + Precalc 42-44 */ + R2(a, b, c, d, e, f, g, h, 36, 0, XW); SCHED_W_0(42, W4, W5, W0, W1, W2, W3); + R2(d, a, b, c, h, e, f, g, 37, 1, XW); SCHED_W_1(42, W4, W5, W0, W1, W2, W3); + R2(c, d, a, b, g, h, e, f, 38, 2, XW); SCHED_W_2(42, W4, W5, W0, W1, W2, W3); + + /* Transform 39-41 + Precalc 45-47 */ + R2(b, c, d, a, f, g, h, e, 39, 0, XW); SCHED_W_0(45, W5, W0, W1, W2, W3, W4); + R2(a, b, c, d, e, f, g, h, 40, 1, XW); SCHED_W_1(45, W5, W0, W1, W2, W3, W4); + R2(d, a, b, c, h, e, f, g, 41, 2, XW); SCHED_W_2(45, W5, W0, W1, W2, W3, W4); + + /* Transform 42-44 + Precalc 48-50 */ + R2(c, d, a, b, g, h, e, f, 42, 0, XW); SCHED_W_0(48, W0, W1, W2, W3, W4, W5); + R2(b, c, d, a, f, g, h, e, 43, 1, XW); SCHED_W_1(48, W0, W1, W2, W3, W4, W5); + R2(a, b, c, d, e, f, g, h, 44, 2, XW); SCHED_W_2(48, W0, W1, W2, W3, W4, W5); + + /* Transform 45-47 + Precalc 51-53 */ + R2(d, a, b, c, h, e, f, g, 45, 0, XW); SCHED_W_0(51, W1, W2, W3, W4, W5, W0); + R2(c, d, a, b, g, h, e, f, 46, 1, XW); SCHED_W_1(51, W1, W2, W3, W4, W5, W0); + R2(b, c, d, a, f, g, h, e, 47, 2, XW); SCHED_W_2(51, W1, W2, W3, W4, W5, W0); + + /* Transform 48-50 + Precalc 54-56 */ + R2(a, b, c, d, e, f, g, h, 48, 0, XW); SCHED_W_0(54, W2, W3, W4, W5, W0, W1); + R2(d, a, b, c, h, e, f, g, 49, 1, XW); SCHED_W_1(54, W2, W3, W4, W5, W0, W1); + R2(c, d, a, b, g, h, e, f, 50, 2, XW); SCHED_W_2(54, W2, W3, W4, W5, W0, W1); + + /* Transform 51-53 + Precalc 57-59 */ + R2(b, c, d, a, f, g, h, e, 51, 0, XW); SCHED_W_0(57, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 52, 1, XW); SCHED_W_1(57, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 53, 2, XW); SCHED_W_2(57, W3, W4, W5, W0, W1, W2); + + /* Transform 54-56 + Precalc 60-62 */ + R2(c, d, a, b, g, h, e, f, 54, 0, XW); SCHED_W_0(60, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 55, 1, XW); SCHED_W_1(60, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 56, 2, XW); SCHED_W_2(60, W4, W5, W0, W1, W2, W3); + + /* Transform 57-59 + Precalc 63 */ + R2(d, a, b, c, h, e, f, g, 57, 0, XW); SCHED_W_0(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 58, 1, XW); + R2(b, c, d, a, f, g, h, e, 59, 2, XW); SCHED_W_1(63, W5, W0, W1, W2, W3, W4); + + /* Transform 60-62 + Precalc 63 */ + R2(a, b, c, d, e, f, g, h, 60, 0, XW); + R2(d, a, b, c, h, e, f, g, 61, 1, XW); SCHED_W_2(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 62, 2, XW); + + /* Transform 63 */ + R2(b, c, d, a, f, g, h, e, 63, 0, XW); + + /* Update the chaining variables. */ + xorl state_h0(RSTATE), a; + xorl state_h1(RSTATE), b; + xorl state_h2(RSTATE), c; + xorl state_h3(RSTATE), d; + movl a, state_h0(RSTATE); + movl b, state_h1(RSTATE); + movl c, state_h2(RSTATE); + movl d, state_h3(RSTATE); + xorl state_h4(RSTATE), e; + xorl state_h5(RSTATE), f; + xorl state_h6(RSTATE), g; + xorl state_h7(RSTATE), h; + movl e, state_h4(RSTATE); + movl f, state_h5(RSTATE); + movl g, state_h6(RSTATE); + movl h, state_h7(RSTATE); + + cmpq $0, RNBLKS; + jne .Loop; + + vzeroall; + + movq (STACK_REG_SAVE + 0 * 8)(%rsp), %rbx; + movq (STACK_REG_SAVE + 1 * 8)(%rsp), %r15; + movq (STACK_REG_SAVE + 2 * 8)(%rsp), %r14; + movq (STACK_REG_SAVE + 3 * 8)(%rsp), %r13; + movq (STACK_REG_SAVE + 4 * 8)(%rsp), %r12; + + vmovdqa %xmm0, IW_W1_ADDR(0, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(0, 0); + vmovdqa %xmm0, IW_W1_ADDR(4, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(4, 0); + vmovdqa %xmm0, IW_W1_ADDR(8, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(8, 0); + + movq %rbp, %rsp; + popq %rbp; + ret; +SYM_FUNC_END(sm3_transform_avx) diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c new file mode 100644 index 000000000000..661b6f22ffcd --- /dev/null +++ b/arch/x86/crypto/sm3_avx_glue.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 Secure Hash Algorithm, AVX assembler accelerated. + * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 + * + * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/types.h> +#include <crypto/sm3.h> +#include <crypto/sm3_base.h> +#include <asm/simd.h> + +asmlinkage void sm3_transform_avx(struct sm3_state *state, + const u8 *data, int nblocks); + +static int sm3_avx_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sm3_state *sctx = shash_desc_ctx(desc); + + if (!crypto_simd_usable() || + (sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) { + sm3_update(sctx, data, len); + return 0; + } + + /* + * Make sure struct sm3_state begins directly with the SM3 + * 256-bit internal state, as this is what the asm functions expect. + */ + BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); + + kernel_fpu_begin(); + sm3_base_do_update(desc, data, len, sm3_transform_avx); + kernel_fpu_end(); + + return 0; +} + +static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!crypto_simd_usable()) { + struct sm3_state *sctx = shash_desc_ctx(desc); + + if (len) + sm3_update(sctx, data, len); + + sm3_final(sctx, out); + return 0; + } + + kernel_fpu_begin(); + if (len) + sm3_base_do_update(desc, data, len, sm3_transform_avx); + sm3_base_do_finalize(desc, sm3_transform_avx); + kernel_fpu_end(); + + return sm3_base_finish(desc, out); +} + +static int sm3_avx_final(struct shash_desc *desc, u8 *out) +{ + if (!crypto_simd_usable()) { + sm3_final(shash_desc_ctx(desc), out); + return 0; + } + + kernel_fpu_begin(); + sm3_base_do_finalize(desc, sm3_transform_avx); + kernel_fpu_end(); + + return sm3_base_finish(desc, out); +} + +static struct shash_alg sm3_avx_alg = { + .digestsize = SM3_DIGEST_SIZE, + .init = sm3_base_init, + .update = sm3_avx_update, + .final = sm3_avx_final, + .finup = sm3_avx_finup, + .descsize = sizeof(struct sm3_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-avx", + .cra_priority = 300, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sm3_avx_mod_init(void) +{ + const char *feature_name; + + if (!boot_cpu_has(X86_FEATURE_AVX)) { + pr_info("AVX instruction are not detected.\n"); + return -ENODEV; + } + + if (!boot_cpu_has(X86_FEATURE_BMI2)) { + pr_info("BMI2 instruction are not detected.\n"); + return -ENODEV; + } + + if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + &feature_name)) { + pr_info("CPU feature '%s' is not supported.\n", feature_name); + return -ENODEV; + } + + return crypto_register_shash(&sm3_avx_alg); +} + +static void __exit sm3_avx_mod_exit(void) +{ + crypto_unregister_shash(&sm3_avx_alg); +} + +module_init(sm3_avx_mod_init); +module_exit(sm3_avx_mod_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); +MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated"); +MODULE_ALIAS_CRYPTO("sm3"); +MODULE_ALIAS_CRYPTO("sm3-avx"); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index e686c5e0537b..eef816fc216d 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2794,7 +2794,7 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re static inline int valid_user_frame(const void __user *fp, unsigned long size) { - return (__range_not_ok(fp, size, TASK_SIZE) == 0); + return __access_ok(fp, size); } static unsigned long get_segment_base(unsigned int segment) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index a3c7ca876aeb..e88791b420ee 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -181,6 +181,27 @@ static struct event_constraint intel_gen_event_constraints[] __read_mostly = EVENT_CONSTRAINT_END }; +static struct event_constraint intel_v5_gen_event_constraints[] __read_mostly = +{ + FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ + FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ + FIXED_EVENT_CONSTRAINT(0x0500, 4), + FIXED_EVENT_CONSTRAINT(0x0600, 5), + FIXED_EVENT_CONSTRAINT(0x0700, 6), + FIXED_EVENT_CONSTRAINT(0x0800, 7), + FIXED_EVENT_CONSTRAINT(0x0900, 8), + FIXED_EVENT_CONSTRAINT(0x0a00, 9), + FIXED_EVENT_CONSTRAINT(0x0b00, 10), + FIXED_EVENT_CONSTRAINT(0x0c00, 11), + FIXED_EVENT_CONSTRAINT(0x0d00, 12), + FIXED_EVENT_CONSTRAINT(0x0e00, 13), + FIXED_EVENT_CONSTRAINT(0x0f00, 14), + FIXED_EVENT_CONSTRAINT(0x1000, 15), + EVENT_CONSTRAINT_END +}; + static struct event_constraint intel_slm_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ @@ -6308,7 +6329,9 @@ __init int intel_pmu_init(void) pr_cont("generic architected perfmon v1, "); name = "generic_arch_v1"; break; - default: + case 2: + case 3: + case 4: /* * default constraints for v2 and up */ @@ -6316,6 +6339,21 @@ __init int intel_pmu_init(void) pr_cont("generic architected perfmon, "); name = "generic_arch_v2+"; break; + default: + /* + * The default constraints for v5 and up can support up to + * 16 fixed counters. For the fixed counters 4 and later, + * the pseudo-encoding is applied. + * The constraints may be cut according to the CPUID enumeration + * by inserting the EVENT_CONSTRAINT_END. + */ + if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) + x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED; + intel_v5_gen_event_constraints[x86_pmu.num_counters_fixed].weight = -1; + x86_pmu.event_constraints = intel_v5_gen_event_constraints; + pr_cont("generic architected perfmon, "); + name = "generic_arch_v5+"; + break; } } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 2e215369df4a..376cc3d66094 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1203,7 +1203,10 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) if (hwc->idx >= INTEL_PMC_IDX_FIXED) { base = MSR_RELOAD_FIXED_CTR0; idx = hwc->idx - INTEL_PMC_IDX_FIXED; - value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx]; + if (x86_pmu.intel_cap.pebs_format < 5) + value = ds->pebs_event_reset[MAX_PEBS_EVENTS_FMT4 + idx]; + else + value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx]; } wrmsrl(base + idx, value); } @@ -1232,8 +1235,12 @@ void intel_pmu_pebs_enable(struct perf_event *event) } } - if (idx >= INTEL_PMC_IDX_FIXED) - idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED); + if (idx >= INTEL_PMC_IDX_FIXED) { + if (x86_pmu.intel_cap.pebs_format < 5) + idx = MAX_PEBS_EVENTS_FMT4 + (idx - INTEL_PMC_IDX_FIXED); + else + idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED); + } /* * Use auto-reload if possible to save a MSR write in the PMI. @@ -2204,6 +2211,7 @@ void __init intel_ds_init(void) break; case 4: + case 5: x86_pmu.drain_pebs = intel_pmu_drain_pebs_icl; x86_pmu.pebs_record_size = sizeof(struct pebs_basic); if (x86_pmu.intel_cap.pebs_baseline) { diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 669c2be14784..fe1742c4ca49 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1329,10 +1329,10 @@ static int branch_map[X86_BR_TYPE_MAP_MAX] = { PERF_BR_SYSCALL, /* X86_BR_SYSCALL */ PERF_BR_SYSRET, /* X86_BR_SYSRET */ PERF_BR_UNKNOWN, /* X86_BR_INT */ - PERF_BR_UNKNOWN, /* X86_BR_IRET */ + PERF_BR_ERET, /* X86_BR_IRET */ PERF_BR_COND, /* X86_BR_JCC */ PERF_BR_UNCOND, /* X86_BR_JMP */ - PERF_BR_UNKNOWN, /* X86_BR_IRQ */ + PERF_BR_IRQ, /* X86_BR_IRQ */ PERF_BR_IND_CALL, /* X86_BR_IND_CALL */ PERF_BR_UNKNOWN, /* X86_BR_ABORT */ PERF_BR_UNKNOWN, /* X86_BR_IN_TX */ diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 2d33bba9a144..82ef87e9a897 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -13,6 +13,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/types.h> +#include <linux/bits.h> +#include <linux/limits.h> #include <linux/slab.h> #include <linux/device.h> @@ -57,6 +59,8 @@ static struct pt_cap_desc { PT_CAP(mtc, 0, CPUID_EBX, BIT(3)), PT_CAP(ptwrite, 0, CPUID_EBX, BIT(4)), PT_CAP(power_event_trace, 0, CPUID_EBX, BIT(5)), + PT_CAP(event_trace, 0, CPUID_EBX, BIT(7)), + PT_CAP(tnt_disable, 0, CPUID_EBX, BIT(8)), PT_CAP(topa_output, 0, CPUID_ECX, BIT(0)), PT_CAP(topa_multiple_entries, 0, CPUID_ECX, BIT(1)), PT_CAP(single_range_output, 0, CPUID_ECX, BIT(2)), @@ -108,6 +112,8 @@ PMU_FORMAT_ATTR(tsc, "config:10" ); PMU_FORMAT_ATTR(noretcomp, "config:11" ); PMU_FORMAT_ATTR(ptw, "config:12" ); PMU_FORMAT_ATTR(branch, "config:13" ); +PMU_FORMAT_ATTR(event, "config:31" ); +PMU_FORMAT_ATTR(notnt, "config:55" ); PMU_FORMAT_ATTR(mtc_period, "config:14-17" ); PMU_FORMAT_ATTR(cyc_thresh, "config:19-22" ); PMU_FORMAT_ATTR(psb_period, "config:24-27" ); @@ -116,6 +122,8 @@ static struct attribute *pt_formats_attr[] = { &format_attr_pt.attr, &format_attr_cyc.attr, &format_attr_pwr_evt.attr, + &format_attr_event.attr, + &format_attr_notnt.attr, &format_attr_fup_on_ptw.attr, &format_attr_mtc.attr, &format_attr_tsc.attr, @@ -296,6 +304,8 @@ fail: RTIT_CTL_CYC_PSB | \ RTIT_CTL_MTC | \ RTIT_CTL_PWR_EVT_EN | \ + RTIT_CTL_EVENT_EN | \ + RTIT_CTL_NOTNT | \ RTIT_CTL_FUP_ON_PTW | \ RTIT_CTL_PTW_EN) @@ -350,6 +360,14 @@ static bool pt_event_valid(struct perf_event *event) !intel_pt_validate_hw_cap(PT_CAP_power_event_trace)) return false; + if (config & RTIT_CTL_EVENT_EN && + !intel_pt_validate_hw_cap(PT_CAP_event_trace)) + return false; + + if (config & RTIT_CTL_NOTNT && + !intel_pt_validate_hw_cap(PT_CAP_tnt_disable)) + return false; + if (config & RTIT_CTL_PTW) { if (!intel_pt_validate_hw_cap(PT_CAP_ptwrite)) return false; @@ -472,7 +490,7 @@ static u64 pt_config_filters(struct perf_event *event) pt->filters.filter[range].msr_b = filter->msr_b; } - rtit_ctl |= filter->config << pt_address_ranges[range].reg_off; + rtit_ctl |= (u64)filter->config << pt_address_ranges[range].reg_off; } return rtit_ctl; @@ -1348,10 +1366,26 @@ static void pt_addr_filters_fini(struct perf_event *event) event->hw.addr_filters = NULL; } -static inline bool valid_kernel_ip(unsigned long ip) +#ifdef CONFIG_X86_64 +/* Clamp to a canonical address greater-than-or-equal-to the address given */ +static u64 clamp_to_ge_canonical_addr(u64 vaddr, u8 vaddr_bits) +{ + return __is_canonical_address(vaddr, vaddr_bits) ? + vaddr : + -BIT_ULL(vaddr_bits - 1); +} + +/* Clamp to a canonical address less-than-or-equal-to the address given */ +static u64 clamp_to_le_canonical_addr(u64 vaddr, u8 vaddr_bits) { - return virt_addr_valid(ip) && kernel_ip(ip); + return __is_canonical_address(vaddr, vaddr_bits) ? + vaddr : + BIT_ULL(vaddr_bits - 1) - 1; } +#else +#define clamp_to_ge_canonical_addr(x, y) (x) +#define clamp_to_le_canonical_addr(x, y) (x) +#endif static int pt_event_addr_filters_validate(struct list_head *filters) { @@ -1367,14 +1401,6 @@ static int pt_event_addr_filters_validate(struct list_head *filters) filter->action == PERF_ADDR_FILTER_ACTION_START) return -EOPNOTSUPP; - if (!filter->path.dentry) { - if (!valid_kernel_ip(filter->offset)) - return -EINVAL; - - if (!valid_kernel_ip(filter->offset + filter->size)) - return -EINVAL; - } - if (++range > intel_pt_validate_hw_cap(PT_CAP_num_address_ranges)) return -EOPNOTSUPP; } @@ -1398,9 +1424,26 @@ static void pt_event_addr_filters_sync(struct perf_event *event) if (filter->path.dentry && !fr[range].start) { msr_a = msr_b = 0; } else { - /* apply the offset */ - msr_a = fr[range].start; - msr_b = msr_a + fr[range].size - 1; + unsigned long n = fr[range].size - 1; + unsigned long a = fr[range].start; + unsigned long b; + + if (a > ULONG_MAX - n) + b = ULONG_MAX; + else + b = a + n; + /* + * Apply the offset. 64-bit addresses written to the + * MSRs must be canonical, but the range can encompass + * non-canonical addresses. Since software cannot + * execute at non-canonical addresses, adjusting to + * canonical addresses does not affect the result of the + * address filter. + */ + msr_a = clamp_to_ge_canonical_addr(a, boot_cpu_data.x86_virt_bits); + msr_b = clamp_to_le_canonical_addr(b, boot_cpu_data.x86_virt_bits); + if (msr_b < msr_a) + msr_a = msr_b = 0; } filters->filter[range].msr_a = msr_a; diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 6ddadb482f68..5fd72d4b8bbb 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -215,10 +215,18 @@ static int parse_discovery_table(struct pci_dev *dev, int die, pci_read_config_dword(dev, bar_offset, &val); - if (val & UNCORE_DISCOVERY_MASK) + if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64) return -EINVAL; - addr = (resource_size_t)(val & ~UNCORE_DISCOVERY_MASK); + addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK); +#ifdef CONFIG_PHYS_ADDR_T_64BIT + if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) { + u32 val2; + + pci_read_config_dword(dev, bar_offset + 4, &val2); + addr |= ((resource_size_t)val2) << 32; + } +#endif size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE; io_addr = ioremap(addr, size); if (!io_addr) @@ -444,7 +452,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = { #define UNCORE_GENERIC_MMIO_SIZE 0x4000 -static unsigned int generic_uncore_mmio_box_ctl(struct intel_uncore_box *box) +static u64 generic_uncore_mmio_box_ctl(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; @@ -456,7 +464,7 @@ static unsigned int generic_uncore_mmio_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { - unsigned int box_ctl = generic_uncore_mmio_box_ctl(box); + u64 box_ctl = generic_uncore_mmio_box_ctl(box); struct intel_uncore_type *type = box->pmu->type; resource_size_t addr; diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index cfaf558bdb6b..f4439357779a 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -18,8 +18,6 @@ #define UNCORE_DISCOVERY_BIR_BASE 0x10 /* Discovery table BAR step */ #define UNCORE_DISCOVERY_BIR_STEP 0x4 -/* Mask of the discovery table offset */ -#define UNCORE_DISCOVERY_MASK 0xf /* Global discovery table size */ #define UNCORE_DISCOVERY_GLOBAL_MAP_SIZE 0x20 diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index 84b87538a15d..bab883c0b6fe 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -22,7 +22,7 @@ #ifdef CONFIG_DEBUG_BUGVERBOSE -#define _BUG_FLAGS(ins, flags) \ +#define _BUG_FLAGS(ins, flags, extra) \ do { \ asm_inline volatile("1:\t" ins "\n" \ ".pushsection __bug_table,\"aw\"\n" \ @@ -31,7 +31,8 @@ do { \ "\t.word %c1" "\t# bug_entry::line\n" \ "\t.word %c2" "\t# bug_entry::flags\n" \ "\t.org 2b+%c3\n" \ - ".popsection" \ + ".popsection\n" \ + extra \ : : "i" (__FILE__), "i" (__LINE__), \ "i" (flags), \ "i" (sizeof(struct bug_entry))); \ @@ -39,14 +40,15 @@ do { \ #else /* !CONFIG_DEBUG_BUGVERBOSE */ -#define _BUG_FLAGS(ins, flags) \ +#define _BUG_FLAGS(ins, flags, extra) \ do { \ asm_inline volatile("1:\t" ins "\n" \ ".pushsection __bug_table,\"aw\"\n" \ "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \ "\t.word %c0" "\t# bug_entry::flags\n" \ "\t.org 2b+%c1\n" \ - ".popsection" \ + ".popsection\n" \ + extra \ : : "i" (flags), \ "i" (sizeof(struct bug_entry))); \ } while (0) @@ -55,7 +57,7 @@ do { \ #else -#define _BUG_FLAGS(ins, flags) asm volatile(ins) +#define _BUG_FLAGS(ins, flags, extra) asm volatile(ins) #endif /* CONFIG_GENERIC_BUG */ @@ -63,8 +65,8 @@ do { \ #define BUG() \ do { \ instrumentation_begin(); \ - _BUG_FLAGS(ASM_UD2, 0); \ - unreachable(); \ + _BUG_FLAGS(ASM_UD2, 0, ""); \ + __builtin_unreachable(); \ } while (0) /* @@ -75,9 +77,9 @@ do { \ */ #define __WARN_FLAGS(flags) \ do { \ + __auto_type f = BUGFLAG_WARNING|(flags); \ instrumentation_begin(); \ - _BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \ - annotate_reachable(); \ + _BUG_FLAGS(ASM_UD2, f, ASM_REACHABLE); \ instrumentation_end(); \ } while (0) diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h new file mode 100644 index 000000000000..3d98c3a60d34 --- /dev/null +++ b/arch/x86/include/asm/coco.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_COCO_H +#define _ASM_X86_COCO_H + +#include <asm/types.h> + +enum cc_vendor { + CC_VENDOR_NONE, + CC_VENDOR_AMD, + CC_VENDOR_HYPERV, + CC_VENDOR_INTEL, +}; + +void cc_set_vendor(enum cc_vendor v); +void cc_set_mask(u64 mask); + +#ifdef CONFIG_ARCH_HAS_CC_PLATFORM +u64 cc_mkenc(u64 val); +u64 cc_mkdec(u64 val); +#else +static inline u64 cc_mkenc(u64 val) +{ + return val; +} + +static inline u64 cc_mkdec(u64 val) +{ + return val; +} +#endif + +#endif /* _ASM_X86_COCO_H */ diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 6db4e2932b3d..3edf05e98e58 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -204,7 +204,7 @@ /* FREE! ( 7*32+10) */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ @@ -299,9 +299,6 @@ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5) /* AVX512 BFLOAT16 instructions */ -#define X86_FEATURE_AMX_BF16 (18*32+22) /* AMX bf16 Support */ -#define X86_FEATURE_AMX_TILE (18*32+24) /* AMX tile Support */ -#define X86_FEATURE_AMX_INT8 (18*32+25) /* AMX int8 Support */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */ @@ -330,6 +327,7 @@ #define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */ #define X86_FEATURE_HWP_EPP (14*32+10) /* HWP Energy Perf. Preference */ #define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */ +#define X86_FEATURE_HFI (14*32+19) /* Hardware Feedback Interface */ /* AMD SVM Feature Identification, CPUID level 0x8000000a (EDX), word 15 */ #define X86_FEATURE_NPT (15*32+ 0) /* Nested Page Table support */ @@ -390,7 +388,10 @@ #define X86_FEATURE_TSXLDTRK (18*32+16) /* TSX Suspend Load Address Tracking */ #define X86_FEATURE_PCONFIG (18*32+18) /* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19) /* Intel ARCH LBR */ +#define X86_FEATURE_AMX_BF16 (18*32+22) /* AMX bf16 Support */ #define X86_FEATURE_AVX512_FP16 (18*32+23) /* AVX512 FP16 */ +#define X86_FEATURE_AMX_TILE (18*32+24) /* AMX tile Support */ +#define X86_FEATURE_AMX_INT8 (18*32+25) /* AMX int8 Support */ #define X86_FEATURE_SPEC_CTRL (18*32+26) /* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27) /* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28) /* Flush L1D cache */ diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h index 3afa990d756b..c5aed9e9226c 100644 --- a/arch/x86/include/asm/cpumask.h +++ b/arch/x86/include/asm/cpumask.h @@ -20,11 +20,21 @@ static __always_inline bool arch_cpu_online(int cpu) { return arch_test_bit(cpu, cpumask_bits(cpu_online_mask)); } + +static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + arch_clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); +} #else static __always_inline bool arch_cpu_online(int cpu) { return cpu == 0; } + +static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + return; +} #endif #define arch_cpu_is_offline(cpu) unlikely(!arch_cpu_online(cpu)) diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 8f28fafa98b3..1231d63f836d 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -56,8 +56,11 @@ # define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31)) #endif -/* Force disable because it's broken beyond repair */ -#define DISABLE_ENQCMD (1 << (X86_FEATURE_ENQCMD & 31)) +#ifdef CONFIG_INTEL_IOMMU_SVM +# define DISABLE_ENQCMD 0 +#else +# define DISABLE_ENQCMD (1 << (X86_FEATURE_ENQCMD & 31)) +#endif #ifdef CONFIG_X86_SGX # define DISABLE_SGX 0 diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index 05a6ab940f45..1b29f58f730f 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -124,7 +124,7 @@ struct insn { #define X86_VEX_B(vex) ((vex) & 0x20) /* VEX3 Byte1 */ #define X86_VEX_L(vex) ((vex) & 0x04) /* VEX3 Byte2, VEX2 Byte1 */ /* VEX bit fields */ -#define X86_EVEX_M(vex) ((vex) & 0x03) /* EVEX Byte1 */ +#define X86_EVEX_M(vex) ((vex) & 0x07) /* EVEX Byte1 */ #define X86_VEX3_M(vex) ((vex) & 0x1f) /* VEX3 Byte1 */ #define X86_VEX2_M 1 /* VEX2.M always 1 */ #define X86_VEX_V(vex) (((vex) & 0x78) >> 3) /* VEX3 Byte2, VEX2 Byte1 */ diff --git a/arch/x86/include/asm/intel_ds.h b/arch/x86/include/asm/intel_ds.h index 8380c3ddd4b2..2f9eeb5c3069 100644 --- a/arch/x86/include/asm/intel_ds.h +++ b/arch/x86/include/asm/intel_ds.h @@ -7,8 +7,9 @@ #define PEBS_BUFFER_SIZE (PAGE_SIZE << 4) /* The maximal number of PEBS events: */ -#define MAX_PEBS_EVENTS 8 -#define MAX_FIXED_PEBS_EVENTS 4 +#define MAX_PEBS_EVENTS_FMT4 8 +#define MAX_PEBS_EVENTS 32 +#define MAX_FIXED_PEBS_EVENTS 16 /* * A debug store configuration. diff --git a/arch/x86/include/asm/intel_pt.h b/arch/x86/include/asm/intel_pt.h index ebe8d2ea44fe..c796e9bc98b6 100644 --- a/arch/x86/include/asm/intel_pt.h +++ b/arch/x86/include/asm/intel_pt.h @@ -13,6 +13,8 @@ enum pt_capabilities { PT_CAP_mtc, PT_CAP_ptwrite, PT_CAP_power_event_trace, + PT_CAP_event_trace, + PT_CAP_tnt_disable, PT_CAP_topa_output, PT_CAP_topa_multiple_entries, PT_CAP_single_range_output, diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6dcccb304775..b61e46743184 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -498,6 +498,7 @@ struct kvm_pmc { bool intr; }; +#define KVM_PMC_MAX_FIXED 3 struct kvm_pmu { unsigned nr_arch_gp_counters; unsigned nr_arch_fixed_counters; @@ -511,7 +512,7 @@ struct kvm_pmu { u64 reserved_bits; u8 version; struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC]; - struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED]; + struct kvm_pmc fixed_counters[KVM_PMC_MAX_FIXED]; struct irq_work irq_work; DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX); @@ -703,7 +704,6 @@ struct kvm_vcpu_arch { struct fpu_guest guest_fpu; u64 xcr0; - u64 guest_supported_xcr0; struct kvm_pio_request pio; void *pio_data; diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 3faf0f97edb1..9f1741ac4769 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -205,6 +205,8 @@ #define RTIT_CTL_DISRETC BIT(11) #define RTIT_CTL_PTW_EN BIT(12) #define RTIT_CTL_BRANCH_EN BIT(13) +#define RTIT_CTL_EVENT_EN BIT(31) +#define RTIT_CTL_NOTNT BIT_ULL(55) #define RTIT_CTL_MTC_RANGE_OFFSET 14 #define RTIT_CTL_MTC_RANGE (0x0full << RTIT_CTL_MTC_RANGE_OFFSET) #define RTIT_CTL_CYC_THRESH_OFFSET 19 @@ -476,6 +478,7 @@ #define MSR_AMD64_ICIBSEXTDCTL 0xc001103c #define MSR_AMD64_IBSOPDATA4 0xc001103d #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ +#define MSR_AMD64_SVM_AVIC_DOORBELL 0xc001011b #define MSR_AMD64_VM_PAGE_FLUSH 0xc001011e #define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 @@ -704,12 +707,14 @@ #define PACKAGE_THERM_STATUS_PROCHOT (1 << 0) #define PACKAGE_THERM_STATUS_POWER_LIMIT (1 << 10) +#define PACKAGE_THERM_STATUS_HFI_UPDATED (1 << 26) #define MSR_IA32_PACKAGE_THERM_INTERRUPT 0x000001b2 #define PACKAGE_THERM_INT_HIGH_ENABLE (1 << 0) #define PACKAGE_THERM_INT_LOW_ENABLE (1 << 1) #define PACKAGE_THERM_INT_PLN_ENABLE (1 << 24) +#define PACKAGE_THERM_INT_HFI_ENABLE (1 << 25) /* Thermal Thresholds Support */ #define THERM_INT_THRESHOLD0_ENABLE (1 << 15) @@ -958,4 +963,8 @@ #define MSR_VM_IGNNE 0xc0010115 #define MSR_VM_HSAVE_PA 0xc0010117 +/* Hardware Feedback Interface */ +#define MSR_IA32_HW_FEEDBACK_PTR 0x17d0 +#define MSR_IA32_HW_FEEDBACK_CONFIG 0x17d1 + #endif /* _ASM_X86_MSR_INDEX_H */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index cc74dc584836..acbaeaf83b61 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -84,7 +84,7 @@ #ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE #else jmp *%\reg #endif @@ -94,7 +94,7 @@ #ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE #else call *%\reg #endif @@ -146,7 +146,7 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; "lfence;\n" \ ANNOTATE_RETPOLINE_SAFE \ "call *%[thunk_target]\n", \ - X86_FEATURE_RETPOLINE_AMD) + X86_FEATURE_RETPOLINE_LFENCE) # define THUNK_TARGET(addr) [thunk_target] "r" (addr) @@ -176,7 +176,7 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; "lfence;\n" \ ANNOTATE_RETPOLINE_SAFE \ "call *%[thunk_target]\n", \ - X86_FEATURE_RETPOLINE_AMD) + X86_FEATURE_RETPOLINE_LFENCE) # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #endif @@ -188,9 +188,11 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; /* The Spectre V2 mitigation variants */ enum spectre_v2_mitigation { SPECTRE_V2_NONE, - SPECTRE_V2_RETPOLINE_GENERIC, - SPECTRE_V2_RETPOLINE_AMD, - SPECTRE_V2_IBRS_ENHANCED, + SPECTRE_V2_RETPOLINE, + SPECTRE_V2_LFENCE, + SPECTRE_V2_EIBRS, + SPECTRE_V2_EIBRS_RETPOLINE, + SPECTRE_V2_EIBRS_LFENCE, }; /* The indirect branch speculation control variants */ diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h index 4d5810c8fab7..9cc82f305f4b 100644 --- a/arch/x86/include/asm/page.h +++ b/arch/x86/include/asm/page.h @@ -71,6 +71,16 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, extern bool __virt_addr_valid(unsigned long kaddr); #define virt_addr_valid(kaddr) __virt_addr_valid((unsigned long) (kaddr)) +static __always_inline u64 __canonical_address(u64 vaddr, u8 vaddr_bits) +{ + return ((s64)vaddr << (64 - vaddr_bits)) >> (64 - vaddr_bits); +} + +static __always_inline u64 __is_canonical_address(u64 vaddr, u8 vaddr_bits) +{ + return __canonical_address(vaddr, vaddr_bits) == vaddr; +} + #endif /* __ASSEMBLY__ */ #include <asm-generic/memory_model.h> diff --git a/arch/x86/include/asm/paravirt_api_clock.h b/arch/x86/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..65ac7cee0dad --- /dev/null +++ b/arch/x86/include/asm/paravirt_api_clock.h @@ -0,0 +1 @@ +#include <asm/paravirt.h> diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index a69012e1903f..e1591467668e 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -279,7 +279,7 @@ extern void (*paravirt_iret)(void); #define paravirt_type(op) \ [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ - [paravirt_opptr] "i" (&(pv_ops.op)) + [paravirt_opptr] "m" (pv_ops.op) #define paravirt_clobber(clobber) \ [paravirt_clobber] "i" (clobber) @@ -316,7 +316,7 @@ int paravirt_disable_iospace(void); */ #define PARAVIRT_CALL \ ANNOTATE_RETPOLINE_SAFE \ - "call *%c[paravirt_opptr];" + "call *%[paravirt_opptr];" /* * These macros are intended to wrap calls through one of the paravirt diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 8fc1b5003713..58d9e4b1fa0a 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -7,7 +7,7 @@ */ #define INTEL_PMC_MAX_GENERIC 32 -#define INTEL_PMC_MAX_FIXED 4 +#define INTEL_PMC_MAX_FIXED 16 #define INTEL_PMC_IDX_FIXED 32 #define X86_PMC_IDX_MAX 64 diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 8a9432fb3802..62ab07e24aef 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -15,17 +15,12 @@ cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS))) \ : (prot)) -/* - * Macros to add or remove encryption attribute - */ -#define pgprot_encrypted(prot) __pgprot(__sme_set(pgprot_val(prot))) -#define pgprot_decrypted(prot) __pgprot(__sme_clr(pgprot_val(prot))) - #ifndef __ASSEMBLY__ #include <linux/spinlock.h> #include <asm/x86_init.h> #include <asm/pkru.h> #include <asm/fpu/api.h> +#include <asm/coco.h> #include <asm-generic/pgtable_uffd.h> #include <linux/page_table_check.h> @@ -38,6 +33,12 @@ void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm, void ptdump_walk_pgd_level_checkwx(void); void ptdump_walk_user_pgd_level_checkwx(void); +/* + * Macros to add or remove encryption attribute + */ +#define pgprot_encrypted(prot) __pgprot(cc_mkenc(pgprot_val(prot))) +#define pgprot_decrypted(prot) __pgprot(cc_mkdec(pgprot_val(prot))) + #ifdef CONFIG_DEBUG_WX #define debug_checkwx() ptdump_walk_pgd_level_checkwx() #define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx() diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index fe5efbcba824..5f6daea1ee24 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -108,16 +108,18 @@ static __always_inline bool should_resched(int preempt_offset) extern asmlinkage void preempt_schedule(void); extern asmlinkage void preempt_schedule_thunk(void); -#define __preempt_schedule_func preempt_schedule_thunk +#define preempt_schedule_dynamic_enabled preempt_schedule_thunk +#define preempt_schedule_dynamic_disabled NULL extern asmlinkage void preempt_schedule_notrace(void); extern asmlinkage void preempt_schedule_notrace_thunk(void); -#define __preempt_schedule_notrace_func preempt_schedule_notrace_thunk +#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk +#define preempt_schedule_notrace_dynamic_disabled NULL #ifdef CONFIG_PREEMPT_DYNAMIC -DECLARE_STATIC_CALL(preempt_schedule, __preempt_schedule_func); +DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); #define __preempt_schedule() \ do { \ @@ -125,7 +127,7 @@ do { \ asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \ } while (0) -DECLARE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func); +DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); #define __preempt_schedule_notrace() \ do { \ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 2c5f12ae7d04..a87e7c33d5ac 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -119,6 +119,8 @@ struct cpuinfo_x86 { int x86_cache_mbm_width_offset; int x86_power; unsigned long loops_per_jiffy; + /* protected processor identification number */ + u64 ppin; /* cpuid returned max cores value: */ u16 x86_max_cores; u16 apicid; diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h index 703663175a5a..4357e0f2cd5f 100644 --- a/arch/x86/include/asm/ptrace.h +++ b/arch/x86/include/asm/ptrace.h @@ -137,7 +137,7 @@ static __always_inline int user_mode(struct pt_regs *regs) #endif } -static inline int v8086_mode(struct pt_regs *regs) +static __always_inline int v8086_mode(struct pt_regs *regs) { #ifdef CONFIG_X86_32 return (regs->flags & X86_VM_MASK); diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index ff0f2d90338a..78ca53512486 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -53,7 +53,6 @@ int set_memory_global(unsigned long addr, int numpages); int set_pages_array_uc(struct page **pages, int addrinarray); int set_pages_array_wc(struct page **pages, int addrinarray); -int set_pages_array_wt(struct page **pages, int addrinarray); int set_pages_array_wb(struct page **pages, int addrinarray); /* @@ -84,7 +83,6 @@ int set_pages_rw(struct page *page, int numpages); int set_direct_map_invalid_noflush(struct page *page); int set_direct_map_default_noflush(struct page *page); bool kernel_page_present(struct page *page); -void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc); extern int kernel_set_to_readonly; diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index b00dbc5fac2b..bb2fb78523ce 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -220,6 +220,42 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_NESTED_CTL_SEV_ENABLE BIT(1) #define SVM_NESTED_CTL_SEV_ES_ENABLE BIT(2) + +/* AVIC */ +#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) +#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 +#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) + +#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) +#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) +#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) +#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) +#define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK (0xFF) + +#define AVIC_DOORBELL_PHYSICAL_ID_MASK (0xFF) + +#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 +#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 +#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF + +enum avic_ipi_failure_cause { + AVIC_IPI_FAILURE_INVALID_INT_TYPE, + AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, + AVIC_IPI_FAILURE_INVALID_TARGET, + AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, +}; + + +/* + * 0xff is broadcast, so the max index allowed for physical APIC ID + * table is 0xfe. APIC IDs above 0xff are reserved. + */ +#define AVIC_MAX_PHYSICAL_ID_COUNT 0xff + +#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) +#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL + + struct vmcb_seg { u16 selector; u16 attrib; diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 2f0b6be8eaab..9619385bf749 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -110,6 +110,7 @@ extern const struct cpumask *cpu_clustergroup_mask(int cpu); #define topology_logical_die_id(cpu) (cpu_data(cpu).logical_die_id) #define topology_die_id(cpu) (cpu_data(cpu).cpu_die_id) #define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) +#define topology_ppin(cpu) (cpu_data(cpu).ppin) extern unsigned int __max_die_per_package; @@ -215,15 +216,26 @@ extern void arch_scale_freq_tick(void); #define arch_scale_freq_tick arch_scale_freq_tick extern void arch_set_max_freq_ratio(bool turbo_disabled); +void init_freq_invariance(bool secondary, bool cppc_ready); #else static inline void arch_set_max_freq_ratio(bool turbo_disabled) { } +static inline void init_freq_invariance(bool secondary, bool cppc_ready) +{ +} #endif -#if defined(CONFIG_ACPI_CPPC_LIB) && defined(CONFIG_SMP) +#ifdef CONFIG_ACPI_CPPC_LIB void init_freq_invariance_cppc(void); -#define init_freq_invariance_cppc init_freq_invariance_cppc +#define arch_init_invariance_cppc init_freq_invariance_cppc + +bool amd_set_max_freq_ratio(u64 *ratio); +#else +static inline bool amd_set_max_freq_ratio(u64 *ratio) +{ + return false; +} #endif #endif /* _ASM_X86_TOPOLOGY_H */ diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index ac96f9b2d64b..f78e2b3501a1 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -12,35 +12,6 @@ #include <asm/smap.h> #include <asm/extable.h> -/* - * Test whether a block of memory is a valid user space address. - * Returns 0 if the range is valid, nonzero otherwise. - */ -static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) -{ - /* - * If we have used "sizeof()" for the size, - * we know it won't overflow the limit (but - * it might overflow the 'addr', so it's - * important to subtract the size from the - * limit, not add it to the address). - */ - if (__builtin_constant_p(size)) - return unlikely(addr > limit - size); - - /* Arbitrary sizes? Be careful about overflow */ - addr += size; - if (unlikely(addr < size)) - return true; - return unlikely(addr > limit); -} - -#define __range_not_ok(addr, size, limit) \ -({ \ - __chk_user_ptr(addr); \ - __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ -}) - #ifdef CONFIG_DEBUG_ATOMIC_SLEEP static inline bool pagefault_disabled(void); # define WARN_ON_IN_IRQ() \ @@ -69,9 +40,11 @@ static inline bool pagefault_disabled(void); #define access_ok(addr, size) \ ({ \ WARN_ON_IN_IRQ(); \ - likely(!__range_not_ok(addr, size, TASK_SIZE_MAX)); \ + likely(__access_ok(addr, size)); \ }) +#include <asm-generic/access_ok.h> + extern int __get_user_1(void); extern int __get_user_2(void); extern int __get_user_4(void); @@ -524,8 +497,6 @@ do { \ unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label); \ } while (0) -#define HAVE_GET_KERNEL_NOFAULT - #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT #define __get_kernel_nofault(dst, src, type, err_label) \ __get_user_size(*((type *)(dst)), (__force type __user *)(src), \ diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 22b7412c08f6..e9170457697e 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -142,6 +142,21 @@ struct x86_init_acpi { }; /** + * struct x86_guest - Functions used by misc guest incarnations like SEV, TDX, etc. + * + * @enc_status_change_prepare Notify HV before the encryption status of a range is changed + * @enc_status_change_finish Notify HV after the encryption status of a range is changed + * @enc_tlb_flush_required Returns true if a TLB flush is needed before changing page encryption status + * @enc_cache_flush_required Returns true if a cache flush is needed before changing page encryption status + */ +struct x86_guest { + void (*enc_status_change_prepare)(unsigned long vaddr, int npages, bool enc); + bool (*enc_status_change_finish)(unsigned long vaddr, int npages, bool enc); + bool (*enc_tlb_flush_required)(bool enc); + bool (*enc_cache_flush_required)(void); +}; + +/** * struct x86_init_ops - functions for platform specific setup * */ @@ -287,6 +302,7 @@ struct x86_platform_ops { struct x86_legacy_features legacy; void (*set_legacy_features)(void); struct x86_hyper_runtime hyper; + struct x86_guest guest; }; struct x86_apic_ops { diff --git a/arch/x86/include/asm/xen/cpuid.h b/arch/x86/include/asm/xen/cpuid.h index a9630104f1c4..78e667a31d6c 100644 --- a/arch/x86/include/asm/xen/cpuid.h +++ b/arch/x86/include/asm/xen/cpuid.h @@ -100,6 +100,13 @@ /* Memory mapped from other domains has valid IOMMU entries */ #define XEN_HVM_CPUID_IOMMU_MAPPINGS (1u << 2) #define XEN_HVM_CPUID_VCPU_ID_PRESENT (1u << 3) /* vcpu id is present in EBX */ +#define XEN_HVM_CPUID_DOMID_PRESENT (1u << 4) /* domid is present in ECX */ +/* + * Bits 55:49 from the IO-APIC RTE and bits 11:5 from the MSI address can be + * used to store high bits for the Destination ID. This expands the Destination + * ID field from 8 to 15 bits, allowing to target APIC IDs up 32768. + */ +#define XEN_HVM_CPUID_EXT_DEST_ID (1u << 5) /* * Leaf 6 (0x40000x05) diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h index 2ee95a7769e6..7b0307acc410 100644 --- a/arch/x86/include/asm/xor.h +++ b/arch/x86/include/asm/xor.h @@ -57,7 +57,8 @@ op(i + 3, 3) static void -xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_sse_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned long lines = bytes >> 8; @@ -108,7 +109,8 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_sse_2_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned long lines = bytes >> 8; @@ -142,8 +144,9 @@ xor_sse_2_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_sse_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned long lines = bytes >> 8; @@ -201,8 +204,9 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_sse_3_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned long lines = bytes >> 8; @@ -238,8 +242,10 @@ xor_sse_3_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_sse_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned long lines = bytes >> 8; @@ -304,8 +310,10 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_sse_4_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned long lines = bytes >> 8; @@ -343,8 +351,11 @@ xor_sse_4_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_sse_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { unsigned long lines = bytes >> 8; @@ -416,8 +427,11 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_sse_5_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { unsigned long lines = bytes >> 8; diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h index 67ceb790e639..7a6b9474591e 100644 --- a/arch/x86/include/asm/xor_32.h +++ b/arch/x86/include/asm/xor_32.h @@ -21,7 +21,8 @@ #include <asm/fpu/api.h> static void -xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned long lines = bytes >> 7; @@ -64,8 +65,9 @@ xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_pII_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned long lines = bytes >> 7; @@ -113,8 +115,10 @@ xor_pII_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_pII_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned long lines = bytes >> 7; @@ -168,8 +172,11 @@ xor_pII_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, static void -xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { unsigned long lines = bytes >> 7; @@ -248,7 +255,8 @@ xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, #undef BLOCK static void -xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned long lines = bytes >> 6; @@ -295,8 +303,9 @@ xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned long lines = bytes >> 6; @@ -352,8 +361,10 @@ xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned long lines = bytes >> 6; @@ -418,8 +429,11 @@ xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { unsigned long lines = bytes >> 6; diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h index 0c4e5b5e3852..7f81dd5897f4 100644 --- a/arch/x86/include/asm/xor_avx.h +++ b/arch/x86/include/asm/xor_avx.h @@ -26,7 +26,8 @@ BLOCK4(8) \ BLOCK4(12) -static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1) +static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1) { unsigned long lines = bytes >> 9; @@ -52,8 +53,9 @@ do { \ kernel_fpu_end(); } -static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1, - unsigned long *p2) +static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2) { unsigned long lines = bytes >> 9; @@ -82,8 +84,10 @@ do { \ kernel_fpu_end(); } -static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1, - unsigned long *p2, unsigned long *p3) +static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { unsigned long lines = bytes >> 9; @@ -115,8 +119,11 @@ do { \ kernel_fpu_end(); } -static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1, - unsigned long *p2, unsigned long *p3, unsigned long *p4) +static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { unsigned long lines = bytes >> 9; diff --git a/arch/x86/include/uapi/asm/shmbuf.h b/arch/x86/include/uapi/asm/shmbuf.h index fce18eaa070c..13775bfdfee2 100644 --- a/arch/x86/include/uapi/asm/shmbuf.h +++ b/arch/x86/include/uapi/asm/shmbuf.h @@ -5,6 +5,10 @@ #if !defined(__x86_64__) || !defined(__ILP32__) #include <asm-generic/shmbuf.h> #else + +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * The shmid64_ds structure for x86 architecture with x32 ABI. * @@ -15,7 +19,7 @@ struct shmid64_ds { struct ipc64_perm shm_perm; /* operation perms */ - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ __kernel_long_t shm_atime; /* last attach time */ __kernel_long_t shm_dtime; /* last detach time */ __kernel_long_t shm_ctime; /* last change time */ diff --git a/arch/x86/include/uapi/asm/signal.h b/arch/x86/include/uapi/asm/signal.h index 164a22a72984..777c3a0f4e23 100644 --- a/arch/x86/include/uapi/asm/signal.h +++ b/arch/x86/include/uapi/asm/signal.h @@ -104,7 +104,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 6aef9ee28a39..6462e3dd98f4 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -21,7 +21,6 @@ CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg CFLAGS_REMOVE_head64.o = -pg CFLAGS_REMOVE_sev.o = -pg -CFLAGS_REMOVE_cc_platform.o = -pg endif KASAN_SANITIZE_head$(BITS).o := n @@ -30,7 +29,6 @@ KASAN_SANITIZE_dumpstack_$(BITS).o := n KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_paravirt.o := n KASAN_SANITIZE_sev.o := n -KASAN_SANITIZE_cc_platform.o := n # With some compiler versions the generated code results in boot hangs, caused # by several compilation units. To be safe, disable all instrumentation. @@ -49,7 +47,6 @@ endif KCOV_INSTRUMENT := n CFLAGS_head$(BITS).o += -fno-stack-protector -CFLAGS_cc_platform.o += -fno-stack-protector CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace @@ -151,8 +148,6 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o -obj-$(CONFIG_ARCH_HAS_CC_PLATFORM) += cc_platform.o - ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/acpi/Makefile b/arch/x86/kernel/acpi/Makefile index cf340d85946a..fc17b3f136fe 100644 --- a/arch/x86/kernel/acpi/Makefile +++ b/arch/x86/kernel/acpi/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_ACPI) += boot.o obj-$(CONFIG_ACPI_SLEEP) += sleep.o wakeup_$(BITS).o obj-$(CONFIG_ACPI_APEI) += apei.o -obj-$(CONFIG_ACPI_CPPC_LIB) += cppc_msr.o +obj-$(CONFIG_ACPI_CPPC_LIB) += cppc.o ifneq ($(CONFIG_ACPI_PROCESSOR),) obj-y += cstate.o diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 5b6d1a95776f..0d01e7f5078c 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -1328,6 +1328,17 @@ static int __init disable_acpi_pci(const struct dmi_system_id *d) return 0; } +static int __init disable_acpi_xsdt(const struct dmi_system_id *d) +{ + if (!acpi_force) { + pr_notice("%s detected: force use of acpi=rsdt\n", d->ident); + acpi_gbl_do_not_use_xsdt = TRUE; + } else { + pr_notice("Warning: DMI blacklist says broken, but acpi XSDT forced\n"); + } + return 0; +} + static int __init dmi_disable_acpi(const struct dmi_system_id *d) { if (!acpi_force) { @@ -1451,6 +1462,19 @@ static const struct dmi_system_id acpi_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 360"), }, }, + /* + * Boxes that need ACPI XSDT use disabled due to corrupted tables + */ + { + .callback = disable_acpi_xsdt, + .ident = "Advantech DAC-BJ01", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "NEC"), + DMI_MATCH(DMI_PRODUCT_NAME, "Bearlake CRB Board"), + DMI_MATCH(DMI_BIOS_VERSION, "V1.12"), + DMI_MATCH(DMI_BIOS_DATE, "02/01/2011"), + }, + }, {} }; diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c new file mode 100644 index 000000000000..df1644d9b3b6 --- /dev/null +++ b/arch/x86/kernel/acpi/cppc.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * cppc.c: CPPC Interface for x86 + * Copyright (c) 2016, Intel Corporation. + */ + +#include <acpi/cppc_acpi.h> +#include <asm/msr.h> +#include <asm/processor.h> +#include <asm/topology.h> + +/* Refer to drivers/acpi/cppc_acpi.c for the description of functions */ + +bool cpc_ffh_supported(void) +{ + return true; +} + +int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) +{ + int err; + + err = rdmsrl_safe_on_cpu(cpunum, reg->address, val); + if (!err) { + u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, + reg->bit_offset); + + *val &= mask; + *val >>= reg->bit_offset; + } + return err; +} + +int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) +{ + u64 rd_val; + int err; + + err = rdmsrl_safe_on_cpu(cpunum, reg->address, &rd_val); + if (!err) { + u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, + reg->bit_offset); + + val <<= reg->bit_offset; + val &= mask; + rd_val &= ~mask; + rd_val |= val; + err = wrmsrl_safe_on_cpu(cpunum, reg->address, rd_val); + } + return err; +} + +bool amd_set_max_freq_ratio(u64 *ratio) +{ + struct cppc_perf_caps perf_caps; + u64 highest_perf, nominal_perf; + u64 perf_ratio; + int rc; + + if (!ratio) + return false; + + rc = cppc_get_perf_caps(0, &perf_caps); + if (rc) { + pr_debug("Could not retrieve perf counters (%d)\n", rc); + return false; + } + + highest_perf = amd_get_highest_perf(); + nominal_perf = perf_caps.nominal_perf; + + if (!highest_perf || !nominal_perf) { + pr_debug("Could not retrieve highest or nominal performance\n"); + return false; + } + + perf_ratio = div_u64(highest_perf * SCHED_CAPACITY_SCALE, nominal_perf); + /* midpoint between max_boost and max_P */ + perf_ratio = (perf_ratio + SCHED_CAPACITY_SCALE) >> 1; + if (!perf_ratio) { + pr_debug("Non-zero highest/nominal perf values led to a 0 ratio\n"); + return false; + } + + *ratio = perf_ratio; + arch_set_max_freq_ratio(false); + + return true; +} + +static DEFINE_MUTEX(freq_invariance_lock); + +void init_freq_invariance_cppc(void) +{ + static bool secondary; + + mutex_lock(&freq_invariance_lock); + + init_freq_invariance(secondary, true); + secondary = true; + + mutex_unlock(&freq_invariance_lock); +} diff --git a/arch/x86/kernel/acpi/cppc_msr.c b/arch/x86/kernel/acpi/cppc_msr.c deleted file mode 100644 index b961de569e7e..000000000000 --- a/arch/x86/kernel/acpi/cppc_msr.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * cppc_msr.c: MSR Interface for CPPC - * Copyright (c) 2016, Intel Corporation. - */ - -#include <acpi/cppc_acpi.h> -#include <asm/msr.h> - -/* Refer to drivers/acpi/cppc_acpi.c for the description of functions */ - -bool cpc_ffh_supported(void) -{ - return true; -} - -int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) -{ - int err; - - err = rdmsrl_safe_on_cpu(cpunum, reg->address, val); - if (!err) { - u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, - reg->bit_offset); - - *val &= mask; - *val >>= reg->bit_offset; - } - return err; -} - -int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) -{ - u64 rd_val; - int err; - - err = rdmsrl_safe_on_cpu(cpunum, reg->address, &rd_val); - if (!err) { - u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, - reg->bit_offset); - - val <<= reg->bit_offset; - val &= mask; - rd_val &= ~mask; - rd_val |= val; - err = wrmsrl_safe_on_cpu(cpunum, reg->address, rd_val); - } - return err; -} diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 1e97f944b47d..3b7f4cdbf2e0 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -15,6 +15,7 @@ #include <asm/desc.h> #include <asm/cacheflush.h> #include <asm/realmode.h> +#include <asm/hypervisor.h> #include <linux/ftrace.h> #include "../../realmode/rm/wakeup.h" @@ -140,9 +141,9 @@ static int __init acpi_sleep_setup(char *str) acpi_realmode_flags |= 4; #ifdef CONFIG_HIBERNATION if (strncmp(str, "s4_hwsig", 8) == 0) - acpi_check_s4_hw_signature(1); + acpi_check_s4_hw_signature = 1; if (strncmp(str, "s4_nohwsig", 10) == 0) - acpi_check_s4_hw_signature(0); + acpi_check_s4_hw_signature = 0; #endif if (strncmp(str, "nonvs", 5) == 0) acpi_nvs_nosave(); @@ -160,3 +161,21 @@ static int __init acpi_sleep_setup(char *str) } __setup("acpi_sleep=", acpi_sleep_setup); + +#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HYPERVISOR_GUEST) +static int __init init_s4_sigcheck(void) +{ + /* + * If running on a hypervisor, honour the ACPI specification + * by default and trigger a clean reboot when the hardware + * signature in FACS is changed after hibernation. + */ + if (acpi_check_s4_hw_signature == -1 && + !hypervisor_is_type(X86_HYPER_NATIVE)) + acpi_check_s4_hw_signature = 1; + + return 0; +} +/* This must happen before acpi_init() which is a subsys initcall */ +arch_initcall(init_s4_sigcheck); +#endif diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 5007c3ffe96f..b4470eabf151 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -389,7 +389,7 @@ static int emit_indirect(int op, int reg, u8 *bytes) * * CALL *%\reg * - * It also tries to inline spectre_v2=retpoline,amd when size permits. + * It also tries to inline spectre_v2=retpoline,lfence when size permits. */ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) { @@ -407,7 +407,7 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) BUG_ON(reg == 4); if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) && - !cpu_feature_enabled(X86_FEATURE_RETPOLINE_AMD)) + !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) return -1; op = insn->opcode.bytes[0]; @@ -438,9 +438,9 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) } /* - * For RETPOLINE_AMD: prepend the indirect CALL/JMP with an LFENCE. + * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. */ - if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_AMD)) { + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { bytes[i++] = 0x0f; bytes[i++] = 0xae; bytes[i++] = 0xe8; /* LFENCE */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 4edb6f0f628c..0c0b09796ced 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -394,35 +394,6 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id; } -static void amd_detect_ppin(struct cpuinfo_x86 *c) -{ - unsigned long long val; - - if (!cpu_has(c, X86_FEATURE_AMD_PPIN)) - return; - - /* When PPIN is defined in CPUID, still need to check PPIN_CTL MSR */ - if (rdmsrl_safe(MSR_AMD_PPIN_CTL, &val)) - goto clear_ppin; - - /* PPIN is locked in disabled mode, clear feature bit */ - if ((val & 3UL) == 1UL) - goto clear_ppin; - - /* If PPIN is disabled, try to enable it */ - if (!(val & 2UL)) { - wrmsrl_safe(MSR_AMD_PPIN_CTL, val | 2UL); - rdmsrl_safe(MSR_AMD_PPIN_CTL, &val); - } - - /* If PPIN_EN bit is 1, return from here; otherwise fall through */ - if (val & 2UL) - return; - -clear_ppin: - clear_cpu_cap(c, X86_FEATURE_AMD_PPIN); -} - u32 amd_get_nodes_per_socket(void) { return nodes_per_socket; @@ -585,6 +556,8 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) * the SME physical address space reduction value. * If BIOS has not enabled SME then don't advertise the * SME feature (set in scattered.c). + * If the kernel has not enabled SME via any means then + * don't advertise the SME feature. * For SEV: If BIOS has not enabled SEV then don't advertise the * SEV and SEV_ES feature (set in scattered.c). * @@ -607,6 +580,9 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) if (IS_ENABLED(CONFIG_X86_32)) goto clear_all; + if (!sme_me_mask) + setup_clear_cpu_cap(X86_FEATURE_SME); + rdmsrl(MSR_K7_HWCR, msr); if (!(msr & MSR_K7_HWCR_SMMLOCK)) goto clear_sev; @@ -947,7 +923,6 @@ static void init_amd(struct cpuinfo_x86 *c) amd_detect_cmp(c); amd_get_topology(c); srat_detect_node(c); - amd_detect_ppin(c); init_amd_cacheinfo(c); diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index 22911deacb6e..9ca008f9e9b1 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -91,7 +91,7 @@ unsigned int aperfmperf_get_khz(int cpu) if (!boot_cpu_has(X86_FEATURE_APERFMPERF)) return 0; - if (!housekeeping_cpu(cpu, HK_FLAG_MISC)) + if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) return 0; if (rcu_is_idle_cpu(cpu)) @@ -114,7 +114,7 @@ void arch_freq_prepare_all(void) return; for_each_online_cpu(cpu) { - if (!housekeeping_cpu(cpu, HK_FLAG_MISC)) + if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) continue; if (rcu_is_idle_cpu(cpu)) continue; /* Idle CPUs are completely uninteresting. */ @@ -136,7 +136,7 @@ unsigned int arch_freq_get_on_cpu(int cpu) if (!boot_cpu_has(X86_FEATURE_APERFMPERF)) return 0; - if (!housekeeping_cpu(cpu, HK_FLAG_MISC)) + if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) return 0; if (aperfmperf_snapshot_cpu(cpu, ktime_get(), true)) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 1c1f218a701d..6296e1ebed1d 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -16,6 +16,7 @@ #include <linux/prctl.h> #include <linux/sched/smt.h> #include <linux/pgtable.h> +#include <linux/bpf.h> #include <asm/spec-ctrl.h> #include <asm/cmdline.h> @@ -650,6 +651,32 @@ static inline const char *spectre_v2_module_string(void) static inline const char *spectre_v2_module_string(void) { return ""; } #endif +#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" +#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" +#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" + +#ifdef CONFIG_BPF_SYSCALL +void unpriv_ebpf_notify(int new_state) +{ + if (new_state) + return; + + /* Unprivileged eBPF is enabled */ + + switch (spectre_v2_enabled) { + case SPECTRE_V2_EIBRS: + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); + break; + case SPECTRE_V2_EIBRS_LFENCE: + if (sched_smt_active()) + pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + break; + default: + break; + } +} +#endif + static inline bool match_option(const char *arg, int arglen, const char *opt) { int len = strlen(opt); @@ -664,7 +691,10 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_FORCE, SPECTRE_V2_CMD_RETPOLINE, SPECTRE_V2_CMD_RETPOLINE_GENERIC, - SPECTRE_V2_CMD_RETPOLINE_AMD, + SPECTRE_V2_CMD_RETPOLINE_LFENCE, + SPECTRE_V2_CMD_EIBRS, + SPECTRE_V2_CMD_EIBRS_RETPOLINE, + SPECTRE_V2_CMD_EIBRS_LFENCE, }; enum spectre_v2_user_cmd { @@ -737,6 +767,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) return SPECTRE_V2_USER_CMD_AUTO; } +static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) +{ + return (mode == SPECTRE_V2_EIBRS || + mode == SPECTRE_V2_EIBRS_RETPOLINE || + mode == SPECTRE_V2_EIBRS_LFENCE); +} + static void __init spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) { @@ -804,7 +841,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) */ if (!boot_cpu_has(X86_FEATURE_STIBP) || !smt_possible || - spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) + spectre_v2_in_eibrs_mode(spectre_v2_enabled)) return; /* @@ -824,9 +861,11 @@ set_mode: static const char * const spectre_v2_strings[] = { [SPECTRE_V2_NONE] = "Vulnerable", - [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", - [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", - [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", + [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", + [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", + [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", + [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", + [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", }; static const struct { @@ -837,8 +876,12 @@ static const struct { { "off", SPECTRE_V2_CMD_NONE, false }, { "on", SPECTRE_V2_CMD_FORCE, true }, { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, - { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, + { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, + { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, + { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, + { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, + { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, { "auto", SPECTRE_V2_CMD_AUTO, false }, }; @@ -875,10 +918,30 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) } if ((cmd == SPECTRE_V2_CMD_RETPOLINE || - cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || - cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && + cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || + cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && !IS_ENABLED(CONFIG_RETPOLINE)) { - pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); + pr_err("%s selected but not compiled in. Switching to AUTO select\n", + mitigation_options[i].option); + return SPECTRE_V2_CMD_AUTO; + } + + if ((cmd == SPECTRE_V2_CMD_EIBRS || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && + !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { + pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", + mitigation_options[i].option); + return SPECTRE_V2_CMD_AUTO; + } + + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && + !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", + mitigation_options[i].option); return SPECTRE_V2_CMD_AUTO; } @@ -887,6 +950,16 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) return cmd; } +static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) +{ + if (!IS_ENABLED(CONFIG_RETPOLINE)) { + pr_err("Kernel not compiled with retpoline; no mitigation available!"); + return SPECTRE_V2_NONE; + } + + return SPECTRE_V2_RETPOLINE; +} + static void __init spectre_v2_select_mitigation(void) { enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); @@ -907,49 +980,64 @@ static void __init spectre_v2_select_mitigation(void) case SPECTRE_V2_CMD_FORCE: case SPECTRE_V2_CMD_AUTO: if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { - mode = SPECTRE_V2_IBRS_ENHANCED; - /* Force it so VMEXIT will restore correctly */ - x86_spec_ctrl_base |= SPEC_CTRL_IBRS; - wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); - goto specv2_set_mode; + mode = SPECTRE_V2_EIBRS; + break; } - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_auto; + + mode = spectre_v2_select_retpoline(); break; - case SPECTRE_V2_CMD_RETPOLINE_AMD: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_amd; + + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: + pr_err(SPECTRE_V2_LFENCE_MSG); + mode = SPECTRE_V2_LFENCE; break; + case SPECTRE_V2_CMD_RETPOLINE_GENERIC: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_generic; + mode = SPECTRE_V2_RETPOLINE; break; + case SPECTRE_V2_CMD_RETPOLINE: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_auto; + mode = spectre_v2_select_retpoline(); + break; + + case SPECTRE_V2_CMD_EIBRS: + mode = SPECTRE_V2_EIBRS; + break; + + case SPECTRE_V2_CMD_EIBRS_LFENCE: + mode = SPECTRE_V2_EIBRS_LFENCE; + break; + + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: + mode = SPECTRE_V2_EIBRS_RETPOLINE; break; } - pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); - return; -retpoline_auto: - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - retpoline_amd: - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { - pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); - goto retpoline_generic; - } - mode = SPECTRE_V2_RETPOLINE_AMD; - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); - } else { - retpoline_generic: - mode = SPECTRE_V2_RETPOLINE_GENERIC; + if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); + + if (spectre_v2_in_eibrs_mode(mode)) { + /* Force it so VMEXIT will restore correctly */ + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); + } + + switch (mode) { + case SPECTRE_V2_NONE: + case SPECTRE_V2_EIBRS: + break; + + case SPECTRE_V2_LFENCE: + case SPECTRE_V2_EIBRS_LFENCE: + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); + fallthrough; + + case SPECTRE_V2_RETPOLINE: + case SPECTRE_V2_EIBRS_RETPOLINE: setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + break; } -specv2_set_mode: spectre_v2_enabled = mode; pr_info("%s\n", spectre_v2_strings[mode]); @@ -975,7 +1063,7 @@ specv2_set_mode: * the CPU supports Enhanced IBRS, kernel might un-intentionally not * enable IBRS around firmware calls. */ - if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { + if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } @@ -1045,6 +1133,10 @@ void cpu_bugs_smt_update(void) { mutex_lock(&spec_ctrl_mutex); + if (sched_smt_active() && unprivileged_ebpf_enabled() && + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) + pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + switch (spectre_v2_user_stibp) { case SPECTRE_V2_USER_NONE: break; @@ -1684,7 +1776,7 @@ static ssize_t tsx_async_abort_show_state(char *buf) static char *stibp_state(void) { - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) return ""; switch (spectre_v2_user_stibp) { @@ -1714,6 +1806,27 @@ static char *ibpb_state(void) return ""; } +static ssize_t spectre_v2_show_state(char *buf) +{ + if (spectre_v2_enabled == SPECTRE_V2_LFENCE) + return sprintf(buf, "Vulnerable: LFENCE\n"); + + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); + + if (sched_smt_active() && unprivileged_ebpf_enabled() && + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) + return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); + + return sprintf(buf, "%s%s%s%s%s%s\n", + spectre_v2_strings[spectre_v2_enabled], + ibpb_state(), + boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", + stibp_state(), + boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", + spectre_v2_module_string()); +} + static ssize_t srbds_show_state(char *buf) { return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); @@ -1739,12 +1852,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); case X86_BUG_SPECTRE_V2: - return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], - ibpb_state(), - boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", - stibp_state(), - boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", - spectre_v2_module_string()); + return spectre_v2_show_state(buf); case X86_BUG_SPEC_STORE_BYPASS: return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 7b8382c11788..64deb7727d00 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -88,6 +88,83 @@ EXPORT_SYMBOL_GPL(get_llc_id); /* L2 cache ID of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id) = BAD_APICID; +static struct ppin_info { + int feature; + int msr_ppin_ctl; + int msr_ppin; +} ppin_info[] = { + [X86_VENDOR_INTEL] = { + .feature = X86_FEATURE_INTEL_PPIN, + .msr_ppin_ctl = MSR_PPIN_CTL, + .msr_ppin = MSR_PPIN + }, + [X86_VENDOR_AMD] = { + .feature = X86_FEATURE_AMD_PPIN, + .msr_ppin_ctl = MSR_AMD_PPIN_CTL, + .msr_ppin = MSR_AMD_PPIN + }, +}; + +static const struct x86_cpu_id ppin_cpuids[] = { + X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]), + X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]), + + /* Legacy models without CPUID enumeration */ + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]), + + {} +}; + +static void ppin_init(struct cpuinfo_x86 *c) +{ + const struct x86_cpu_id *id; + unsigned long long val; + struct ppin_info *info; + + id = x86_match_cpu(ppin_cpuids); + if (!id) + return; + + /* + * Testing the presence of the MSR is not enough. Need to check + * that the PPIN_CTL allows reading of the PPIN. + */ + info = (struct ppin_info *)id->driver_data; + + if (rdmsrl_safe(info->msr_ppin_ctl, &val)) + goto clear_ppin; + + if ((val & 3UL) == 1UL) { + /* PPIN locked in disabled mode */ + goto clear_ppin; + } + + /* If PPIN is disabled, try to enable */ + if (!(val & 2UL)) { + wrmsrl_safe(info->msr_ppin_ctl, val | 2UL); + rdmsrl_safe(info->msr_ppin_ctl, &val); + } + + /* Is the enable bit set? */ + if (val & 2UL) { + c->ppin = __rdmsr(info->msr_ppin); + set_cpu_cap(c, info->feature); + return; + } + +clear_ppin: + clear_cpu_cap(c, info->feature); +} + /* correctly size the local cpu masks */ void __init setup_cpu_local_masks(void) { @@ -1655,6 +1732,8 @@ static void identify_cpu(struct cpuinfo_x86 *c) c->x86_capability[i] |= boot_cpu_data.x86_capability[i]; } + ppin_init(c); + /* Init Machine Check Exception if available. */ mcheck_cpu_init(c); diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 9f4b508886dd..1940d305db1c 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -993,6 +993,7 @@ static struct attribute *default_attrs[] = { NULL, /* possibly interrupt_enable if supported, see below */ NULL, }; +ATTRIBUTE_GROUPS(default); #define to_block(k) container_of(k, struct threshold_block, kobj) #define to_attr(a) container_of(a, struct threshold_attr, attr) @@ -1029,7 +1030,7 @@ static void threshold_block_release(struct kobject *kobj); static struct kobj_type threshold_ktype = { .sysfs_ops = &threshold_ops, - .default_attrs = default_attrs, + .default_groups = default_groups, .release = threshold_block_release, }; @@ -1101,10 +1102,10 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb b->threshold_limit = THRESHOLD_MAX; if (b->interrupt_capable) { - threshold_ktype.default_attrs[2] = &interrupt_enable.attr; + default_attrs[2] = &interrupt_enable.attr; b->interrupt_enable = 1; } else { - threshold_ktype.default_attrs[2] = NULL; + default_attrs[2] = NULL; } INIT_LIST_HEAD(&b->miscj); diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 5818b837fd4d..e53d3e6096fb 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -138,12 +138,7 @@ void mce_setup(struct mce *m) m->socketid = cpu_data(m->extcpu).phys_proc_id; m->apicid = cpu_data(m->extcpu).initial_apicid; m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); - - if (this_cpu_has(X86_FEATURE_INTEL_PPIN)) - m->ppin = __rdmsr(MSR_PPIN); - else if (this_cpu_has(X86_FEATURE_AMD_PPIN)) - m->ppin = __rdmsr(MSR_AMD_PPIN); - + m->ppin = cpu_data(m->extcpu).ppin; m->microcode = boot_cpu_data.microcode; } @@ -1304,10 +1299,12 @@ static void kill_me_maybe(struct callback_head *cb) /* * -EHWPOISON from memory_failure() means that it already sent SIGBUS - * to the current process with the proper error info, so no need to - * send SIGBUS here again. + * to the current process with the proper error info, + * -EOPNOTSUPP means hwpoison_filter() filtered the error event, + * + * In both cases, no further processing is required. */ - if (ret == -EHWPOISON) + if (ret == -EHWPOISON || ret == -EOPNOTSUPP) return; pr_err("Memory error not recovered"); diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index baafbb37be67..95275a5e57e0 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -470,47 +470,6 @@ void intel_clear_lmce(void) wrmsrl(MSR_IA32_MCG_EXT_CTL, val); } -static void intel_ppin_init(struct cpuinfo_x86 *c) -{ - unsigned long long val; - - /* - * Even if testing the presence of the MSR would be enough, we don't - * want to risk the situation where other models reuse this MSR for - * other purposes. - */ - switch (c->x86_model) { - case INTEL_FAM6_IVYBRIDGE_X: - case INTEL_FAM6_HASWELL_X: - case INTEL_FAM6_BROADWELL_D: - case INTEL_FAM6_BROADWELL_X: - case INTEL_FAM6_SKYLAKE_X: - case INTEL_FAM6_ICELAKE_X: - case INTEL_FAM6_ICELAKE_D: - case INTEL_FAM6_SAPPHIRERAPIDS_X: - case INTEL_FAM6_XEON_PHI_KNL: - case INTEL_FAM6_XEON_PHI_KNM: - - if (rdmsrl_safe(MSR_PPIN_CTL, &val)) - return; - - if ((val & 3UL) == 1UL) { - /* PPIN locked in disabled mode */ - return; - } - - /* If PPIN is disabled, try to enable */ - if (!(val & 2UL)) { - wrmsrl_safe(MSR_PPIN_CTL, val | 2UL); - rdmsrl_safe(MSR_PPIN_CTL, &val); - } - - /* Is the enable bit set? */ - if (val & 2UL) - set_cpu_cap(c, X86_FEATURE_INTEL_PPIN); - } -} - /* * Enable additional error logs from the integrated * memory controller on processors that support this. @@ -535,7 +494,6 @@ void mce_intel_feature_init(struct cpuinfo_x86 *c) { intel_init_cmci(); intel_init_lmce(); - intel_ppin_init(c); intel_imc_init(c); } diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 5a99f993e639..e0a572472052 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -33,6 +33,7 @@ #include <asm/nmi.h> #include <clocksource/hyperv_timer.h> #include <asm/numa.h> +#include <asm/coco.h> /* Is Linux running as the root partition? */ bool hv_root_partition; @@ -344,6 +345,11 @@ static void __init ms_hyperv_init_platform(void) */ swiotlb_force = SWIOTLB_FORCE; #endif + /* Isolation VMs are unenlightened SEV-based VMs, thus this check: */ + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + if (hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE) + cc_set_vendor(CC_VENDOR_HYPERV); + } } if (hv_max_functions_eax >= HYPERV_CPUID_NESTED_FEATURES) { diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 21d1f062895a..4143b1e4c5c6 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -26,6 +26,7 @@ struct cpuid_bit { static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_APERFMPERF, CPUID_ECX, 0, 0x00000006, 0 }, { X86_FEATURE_EPB, CPUID_ECX, 3, 0x00000006, 0 }, + { X86_FEATURE_INTEL_PPIN, CPUID_EBX, 0, 0x00000007, 1 }, { X86_FEATURE_CQM_LLC, CPUID_EDX, 1, 0x0000000f, 0 }, { X86_FEATURE_CQM_OCCUP_LLC, CPUID_EDX, 0, 0x0000000f, 1 }, { X86_FEATURE_CQM_MBM_TOTAL, CPUID_EDX, 1, 0x0000000f, 1 }, diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index 001808e3901c..7c63a1911fae 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -13,6 +13,30 @@ #include "sgx.h" /* + * Calculate byte offset of a PCMD struct associated with an enclave page. PCMD's + * follow right after the EPC data in the backing storage. In addition to the + * visible enclave pages, there's one extra page slot for SECS, before PCMD + * structs. + */ +static inline pgoff_t sgx_encl_get_backing_page_pcmd_offset(struct sgx_encl *encl, + unsigned long page_index) +{ + pgoff_t epc_end_off = encl->size + sizeof(struct sgx_secs); + + return epc_end_off + page_index * sizeof(struct sgx_pcmd); +} + +/* + * Free a page from the backing storage in the given page index. + */ +static inline void sgx_encl_truncate_backing_page(struct sgx_encl *encl, unsigned long page_index) +{ + struct inode *inode = file_inode(encl->backing); + + shmem_truncate_range(inode, PFN_PHYS(page_index), PFN_PHYS(page_index) + PAGE_SIZE - 1); +} + +/* * ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC * Pages" in the SDM. */ @@ -22,9 +46,11 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page, { unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK; struct sgx_encl *encl = encl_page->encl; + pgoff_t page_index, page_pcmd_off; struct sgx_pageinfo pginfo; struct sgx_backing b; - pgoff_t page_index; + bool pcmd_page_empty; + u8 *pcmd_page; int ret; if (secs_page) @@ -32,14 +58,16 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page, else page_index = PFN_DOWN(encl->size); + page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index); + ret = sgx_encl_get_backing(encl, page_index, &b); if (ret) return ret; pginfo.addr = encl_page->desc & PAGE_MASK; pginfo.contents = (unsigned long)kmap_atomic(b.contents); - pginfo.metadata = (unsigned long)kmap_atomic(b.pcmd) + - b.pcmd_offset; + pcmd_page = kmap_atomic(b.pcmd); + pginfo.metadata = (unsigned long)pcmd_page + b.pcmd_offset; if (secs_page) pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page); @@ -55,11 +83,24 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page, ret = -EFAULT; } - kunmap_atomic((void *)(unsigned long)(pginfo.metadata - b.pcmd_offset)); + memset(pcmd_page + b.pcmd_offset, 0, sizeof(struct sgx_pcmd)); + + /* + * The area for the PCMD in the page was zeroed above. Check if the + * whole page is now empty meaning that all PCMD's have been zeroed: + */ + pcmd_page_empty = !memchr_inv(pcmd_page, 0, PAGE_SIZE); + + kunmap_atomic(pcmd_page); kunmap_atomic((void *)(unsigned long)pginfo.contents); sgx_encl_put_backing(&b, false); + sgx_encl_truncate_backing_page(encl, page_index); + + if (pcmd_page_empty) + sgx_encl_truncate_backing_page(encl, PFN_DOWN(page_pcmd_off)); + return ret; } @@ -410,6 +451,8 @@ void sgx_encl_release(struct kref *ref) } kfree(entry); + /* Invoke scheduler to prevent soft lockups. */ + cond_resched(); } xa_destroy(&encl->page_array); @@ -577,7 +620,7 @@ static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, struct sgx_backing *backing) { - pgoff_t pcmd_index = PFN_DOWN(encl->size) + 1 + (page_index >> 5); + pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index); struct page *contents; struct page *pcmd; @@ -585,7 +628,7 @@ int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, if (IS_ERR(contents)) return PTR_ERR(contents); - pcmd = sgx_encl_get_backing_page(encl, pcmd_index); + pcmd = sgx_encl_get_backing_page(encl, PFN_DOWN(page_pcmd_off)); if (IS_ERR(pcmd)) { put_page(contents); return PTR_ERR(pcmd); @@ -594,9 +637,7 @@ int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, backing->page_index = page_index; backing->contents = contents; backing->pcmd = pcmd; - backing->pcmd_offset = - (page_index & (PAGE_SIZE / sizeof(struct sgx_pcmd) - 1)) * - sizeof(struct sgx_pcmd); + backing->pcmd_offset = page_pcmd_off & (PAGE_SIZE - 1); return 0; } diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 4b41efc9e367..8e4bc6453d26 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -344,10 +344,8 @@ static void sgx_reclaim_pages(void) { struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; struct sgx_backing backing[SGX_NR_TO_SCAN]; - struct sgx_epc_section *section; struct sgx_encl_page *encl_page; struct sgx_epc_page *epc_page; - struct sgx_numa_node *node; pgoff_t page_index; int cnt = 0; int ret; @@ -418,13 +416,7 @@ skip: kref_put(&encl_page->encl->refcount, sgx_encl_release); epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; - section = &sgx_epc_sections[epc_page->section]; - node = section->node; - - spin_lock(&node->lock); - list_add_tail(&epc_page->list, &node->free_page_list); - spin_unlock(&node->lock); - atomic_long_inc(&sgx_nr_free_pages); + sgx_free_epc_page(epc_page); } } diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 53de044e5654..afae4dd77495 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -81,12 +81,6 @@ static int copy_code(struct pt_regs *regs, u8 *buf, unsigned long src, /* The user space code from other tasks cannot be accessed. */ if (regs != task_pt_regs(current)) return -EPERM; - /* - * Make sure userspace isn't trying to trick us into dumping kernel - * memory by pointing the userspace instruction pointer at it. - */ - if (__chk_range_not_ok(src, nbytes, TASK_SIZE_MAX)) - return -EINVAL; /* * Even if named copy_from_user_nmi() this can be invoked from diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index bc0657f0deed..f267205f2d5a 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -995,8 +995,10 @@ early_param("memmap", parse_memmap_opt); */ void __init e820__reserve_setup_data(void) { + struct setup_indirect *indirect; struct setup_data *data; - u64 pa_data; + u64 pa_data, pa_next; + u32 len; pa_data = boot_params.hdr.setup_data; if (!pa_data) @@ -1004,6 +1006,14 @@ void __init e820__reserve_setup_data(void) while (pa_data) { data = early_memremap(pa_data, sizeof(*data)); + if (!data) { + pr_warn("e820: failed to memremap setup_data entry\n"); + return; + } + + len = sizeof(*data); + pa_next = data->next; + e820__range_update(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); /* @@ -1015,18 +1025,27 @@ void __init e820__reserve_setup_data(void) sizeof(*data) + data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { - e820__range_update(((struct setup_indirect *)data->data)->addr, - ((struct setup_indirect *)data->data)->len, - E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); - e820__range_update_kexec(((struct setup_indirect *)data->data)->addr, - ((struct setup_indirect *)data->data)->len, - E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); + if (data->type == SETUP_INDIRECT) { + len += data->len; + early_memunmap(data, sizeof(*data)); + data = early_memremap(pa_data, len); + if (!data) { + pr_warn("e820: failed to memremap indirect setup_data\n"); + return; + } + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) { + e820__range_update(indirect->addr, indirect->len, + E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); + e820__range_update_kexec(indirect->addr, indirect->len, + E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); + } } - pa_data = data->next; - early_memunmap(data, sizeof(*data)); + pa_data = pa_next; + early_memunmap(data, len); } e820__update_table(e820_table); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 8dea01ffc5c1..19821f027cb3 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -612,6 +612,13 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags) fpu_inherit_perms(dst_fpu); fpregs_unlock(); + /* + * Children never inherit PASID state. + * Force it to have its init value: + */ + if (use_xsave()) + dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID; + trace_x86_fpu_copy_src(src_fpu); trace_x86_fpu_copy_dst(dst_fpu); diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c index 437d7c930c0b..75ffaef8c299 100644 --- a/arch/x86/kernel/fpu/regset.c +++ b/arch/x86/kernel/fpu/regset.c @@ -91,11 +91,9 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { struct fpu *fpu = &target->thread.fpu; - struct user32_fxsr_struct newstate; + struct fxregs_state newstate; int ret; - BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state)); - if (!cpu_feature_enabled(X86_FEATURE_FXSR)) return -ENODEV; @@ -116,9 +114,10 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, /* Copy the state */ memcpy(&fpu->fpstate->regs.fxsave, &newstate, sizeof(newstate)); - /* Clear xmm8..15 */ + /* Clear xmm8..15 for 32-bit callers */ BUILD_BUG_ON(sizeof(fpu->__fpstate.regs.fxsave.xmm_space) != 16 * 16); - memset(&fpu->fpstate->regs.fxsave.xmm_space[8], 0, 8 * 16); + if (in_ia32_syscall()) + memset(&fpu->fpstate->regs.fxsave.xmm_space[8*4], 0, 8 * 16); /* Mark FP and SSE as in use when XSAVE is enabled */ if (use_xsave()) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 02b3ddaf4f75..7c7824ae7862 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1558,7 +1558,10 @@ static int fpstate_realloc(u64 xfeatures, unsigned int ksize, fpregs_restore_userregs(); newfps->xfeatures = curfps->xfeatures | xfeatures; - newfps->user_xfeatures = curfps->user_xfeatures | xfeatures; + + if (!guest_fpu) + newfps->user_xfeatures = curfps->user_xfeatures | xfeatures; + newfps->xfd = curfps->xfd & ~xfeatures; /* Do the final updates within the locked region */ diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index de563db9cdcd..4f5ecbbaae77 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -126,7 +126,7 @@ static bool __head check_la57_support(unsigned long physaddr) } #endif -static unsigned long sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd) +static unsigned long __head sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd) { unsigned long vaddr, vaddr_end; int i; diff --git a/arch/x86/kernel/kdebugfs.c b/arch/x86/kernel/kdebugfs.c index 64b6da95af98..e2e89bebcbc3 100644 --- a/arch/x86/kernel/kdebugfs.c +++ b/arch/x86/kernel/kdebugfs.c @@ -88,11 +88,13 @@ create_setup_data_node(struct dentry *parent, int no, static int __init create_setup_data_nodes(struct dentry *parent) { + struct setup_indirect *indirect; struct setup_data_node *node; struct setup_data *data; - int error; + u64 pa_data, pa_next; struct dentry *d; - u64 pa_data; + int error; + u32 len; int no = 0; d = debugfs_create_dir("setup_data", parent); @@ -112,12 +114,29 @@ static int __init create_setup_data_nodes(struct dentry *parent) error = -ENOMEM; goto err_dir; } - - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { - node->paddr = ((struct setup_indirect *)data->data)->addr; - node->type = ((struct setup_indirect *)data->data)->type; - node->len = ((struct setup_indirect *)data->data)->len; + pa_next = data->next; + + if (data->type == SETUP_INDIRECT) { + len = sizeof(*data) + data->len; + memunmap(data); + data = memremap(pa_data, len, MEMREMAP_WB); + if (!data) { + kfree(node); + error = -ENOMEM; + goto err_dir; + } + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) { + node->paddr = indirect->addr; + node->type = indirect->type; + node->len = indirect->len; + } else { + node->paddr = pa_data; + node->type = data->type; + node->len = data->len; + } } else { node->paddr = pa_data; node->type = data->type; @@ -125,7 +144,7 @@ static int __init create_setup_data_nodes(struct dentry *parent) } create_setup_data_node(d, no, node); - pa_data = data->next; + pa_data = pa_next; memunmap(data); no++; diff --git a/arch/x86/kernel/ksysfs.c b/arch/x86/kernel/ksysfs.c index d0a19121c6a4..257892fcefa7 100644 --- a/arch/x86/kernel/ksysfs.c +++ b/arch/x86/kernel/ksysfs.c @@ -91,26 +91,41 @@ static int get_setup_data_paddr(int nr, u64 *paddr) static int __init get_setup_data_size(int nr, size_t *size) { - int i = 0; + u64 pa_data = boot_params.hdr.setup_data, pa_next; + struct setup_indirect *indirect; struct setup_data *data; - u64 pa_data = boot_params.hdr.setup_data; + int i = 0; + u32 len; while (pa_data) { data = memremap(pa_data, sizeof(*data), MEMREMAP_WB); if (!data) return -ENOMEM; + pa_next = data->next; + if (nr == i) { - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) - *size = ((struct setup_indirect *)data->data)->len; - else + if (data->type == SETUP_INDIRECT) { + len = sizeof(*data) + data->len; + memunmap(data); + data = memremap(pa_data, len, MEMREMAP_WB); + if (!data) + return -ENOMEM; + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) + *size = indirect->len; + else + *size = data->len; + } else { *size = data->len; + } memunmap(data); return 0; } - pa_data = data->next; + pa_data = pa_next; memunmap(data); i++; } @@ -120,9 +135,11 @@ static int __init get_setup_data_size(int nr, size_t *size) static ssize_t type_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { + struct setup_indirect *indirect; + struct setup_data *data; int nr, ret; u64 paddr; - struct setup_data *data; + u32 len; ret = kobj_to_setup_data_nr(kobj, &nr); if (ret) @@ -135,10 +152,20 @@ static ssize_t type_show(struct kobject *kobj, if (!data) return -ENOMEM; - if (data->type == SETUP_INDIRECT) - ret = sprintf(buf, "0x%x\n", ((struct setup_indirect *)data->data)->type); - else + if (data->type == SETUP_INDIRECT) { + len = sizeof(*data) + data->len; + memunmap(data); + data = memremap(paddr, len, MEMREMAP_WB); + if (!data) + return -ENOMEM; + + indirect = (struct setup_indirect *)data->data; + + ret = sprintf(buf, "0x%x\n", indirect->type); + } else { ret = sprintf(buf, "0x%x\n", data->type); + } + memunmap(data); return ret; } @@ -149,9 +176,10 @@ static ssize_t setup_data_data_read(struct file *fp, char *buf, loff_t off, size_t count) { + struct setup_indirect *indirect; + struct setup_data *data; int nr, ret = 0; u64 paddr, len; - struct setup_data *data; void *p; ret = kobj_to_setup_data_nr(kobj, &nr); @@ -165,10 +193,27 @@ static ssize_t setup_data_data_read(struct file *fp, if (!data) return -ENOMEM; - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { - paddr = ((struct setup_indirect *)data->data)->addr; - len = ((struct setup_indirect *)data->data)->len; + if (data->type == SETUP_INDIRECT) { + len = sizeof(*data) + data->len; + memunmap(data); + data = memremap(paddr, len, MEMREMAP_WB); + if (!data) + return -ENOMEM; + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) { + paddr = indirect->addr; + len = indirect->len; + } else { + /* + * Even though this is technically undefined, return + * the data as though it is a normal setup_data struct. + * This will at least allow it to be inspected. + */ + paddr += sizeof(*data); + len = data->len; + } } else { paddr += sizeof(*data); len = data->len; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index a438217cbfac..d77481ecb0d5 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -462,19 +462,24 @@ static bool pv_tlb_flush_supported(void) { return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && !kvm_para_has_hint(KVM_HINTS_REALTIME) && - kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)); + kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && + !boot_cpu_has(X86_FEATURE_MWAIT) && + (num_possible_cpus() != 1)); } static bool pv_ipi_supported(void) { - return kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI); + return (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI) && + (num_possible_cpus() != 1)); } static bool pv_sched_yield_supported(void) { return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) && !kvm_para_has_hint(KVM_HINTS_REALTIME) && - kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)); + kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && + !boot_cpu_has(X86_FEATURE_MWAIT) && + (num_possible_cpus() != 1)); } #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG) @@ -619,7 +624,7 @@ static void kvm_smp_send_call_func_ipi(const struct cpumask *mask) /* Make sure other vCPUs get a chance to run if they need to. */ for_each_cpu(cpu, mask) { - if (vcpu_is_preempted(cpu)) { + if (!idle_cpu(cpu) && vcpu_is_preempted(cpu)) { kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu)); break; } diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index a35cbf9107af..c5caa7311bd8 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -239,6 +239,9 @@ static void __init kvmclock_init_mem(void) static int __init kvm_setup_vsyscall_timeinfo(void) { + if (!kvm_para_available() || !kvmclock) + return 0; + kvmclock_init_mem(); #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 95fa745e310a..96d7c27b7093 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -273,6 +273,14 @@ int module_finalize(const Elf_Ehdr *hdr, retpolines = s; } + /* + * See alternative_instructions() for the ordering rules between the + * various patching types. + */ + if (para) { + void *pseg = (void *)para->sh_addr; + apply_paravirt(pseg, pseg + para->sh_size); + } if (retpolines) { void *rseg = (void *)retpolines->sh_addr; apply_retpolines(rseg, rseg + retpolines->sh_size); @@ -290,11 +298,6 @@ int module_finalize(const Elf_Ehdr *hdr, tseg, tseg + text->sh_size); } - if (para) { - void *pseg = (void *)para->sh_addr; - apply_paravirt(pseg, pseg + para->sh_size); - } - /* make jump label nops */ jump_label_apply_nops(me); diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 4bce802d25fb..e73f7df362f5 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -292,7 +292,6 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n", reason, smp_processor_id()); - pr_emerg("Do you have a strange power saving mode enabled?\n"); if (unknown_nmi_panic || panic_on_unrecovered_nmi) nmi_panic(regs, "NMI: Not continuing"); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 81d8ef036637..e131d71b3cae 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -765,8 +765,11 @@ void stop_this_cpu(void *dummy) * without the encryption bit, they don't race each other when flushed * and potentially end up with the wrong entry being committed to * memory. + * + * Test the CPUID bit directly because the machine might've cleared + * X86_FEATURE_SME due to cmdline options. */ - if (boot_cpu_has(X86_FEATURE_SME)) + if (cpuid_eax(0x8000001f) & BIT(0)) native_wbinvd(); for (;;) { /* diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 6d2244c94799..8d2f2f995539 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -1224,7 +1224,7 @@ static struct user_regset x86_64_regsets[] __ro_after_init = { }, [REGSET_FP] = { .core_note_type = NT_PRFPREG, - .n = sizeof(struct user_i387_struct) / sizeof(long), + .n = sizeof(struct fxregs_state) / sizeof(long), .size = sizeof(long), .align = sizeof(long), .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set }, @@ -1271,7 +1271,7 @@ static struct user_regset x86_32_regsets[] __ro_after_init = { }, [REGSET_XFP] = { .core_note_type = NT_PRXFPREG, - .n = sizeof(struct user32_fxsr_struct) / sizeof(u32), + .n = sizeof(struct fxregs_state) / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set }, diff --git a/arch/x86/kernel/resource.c b/arch/x86/kernel/resource.c index 9ae64f9af956..9b9fb7882c20 100644 --- a/arch/x86/kernel/resource.c +++ b/arch/x86/kernel/resource.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/dmi.h> #include <linux/ioport.h> #include <asm/e820/api.h> @@ -24,31 +23,11 @@ static void resource_clip(struct resource *res, resource_size_t start, res->start = end + 1; } -/* - * Some BIOS-es contain a bug where they add addresses which map to - * system RAM in the PCI host bridge window returned by the ACPI _CRS - * method, see commit 4dc2287c1805 ("x86: avoid E820 regions when - * allocating address space"). To avoid this Linux by default excludes - * E820 reservations when allocating addresses since 2010. - * In 2019 some systems have shown-up with E820 reservations which cover - * the entire _CRS returned PCI host bridge window, causing all attempts - * to assign memory to PCI BARs to fail if Linux uses E820 reservations. - * - * Ideally Linux would fully stop using E820 reservations, but then - * the old systems this was added for will regress. - * Instead keep the old behavior for old systems, while ignoring the - * E820 reservations for any systems from now on. - */ static void remove_e820_regions(struct resource *avail) { - int i, year = dmi_get_bios_year(); + int i; struct e820_entry *entry; - if (year >= 2018) - return; - - pr_info_once("PCI: Removing E820 reservations from host bridge windows\n"); - for (i = 0; i < e820_table->nr_entries; i++) { entry = &e820_table->entries[i]; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index f7a132eb794d..90d7e1788c91 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -369,21 +369,41 @@ static void __init parse_setup_data(void) static void __init memblock_x86_reserve_range_setup_data(void) { + struct setup_indirect *indirect; struct setup_data *data; - u64 pa_data; + u64 pa_data, pa_next; + u32 len; pa_data = boot_params.hdr.setup_data; while (pa_data) { data = early_memremap(pa_data, sizeof(*data)); + if (!data) { + pr_warn("setup: failed to memremap setup_data entry\n"); + return; + } + + len = sizeof(*data); + pa_next = data->next; + memblock_reserve(pa_data, sizeof(*data) + data->len); - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) - memblock_reserve(((struct setup_indirect *)data->data)->addr, - ((struct setup_indirect *)data->data)->len); + if (data->type == SETUP_INDIRECT) { + len += data->len; + early_memunmap(data, sizeof(*data)); + data = early_memremap(pa_data, len); + if (!data) { + pr_warn("setup: failed to memremap indirect setup_data\n"); + return; + } - pa_data = data->next; - early_memunmap(data, sizeof(*data)); + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) + memblock_reserve(indirect->addr, indirect->len); + } + + pa_data = pa_next; + early_memunmap(data, len); } } diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 617012f4619f..2ef14772dc04 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -83,10 +83,6 @@ #include <asm/hw_irq.h> #include <asm/stackprotector.h> -#ifdef CONFIG_ACPI_CPPC_LIB -#include <acpi/cppc_acpi.h> -#endif - /* representing HT siblings of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); @@ -155,8 +151,6 @@ static inline void smpboot_restore_warm_reset_vector(void) *((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0; } -static void init_freq_invariance(bool secondary, bool cppc_ready); - /* * Report back to the Boot Processor during boot time or to the caller processor * during CPU online. @@ -2097,48 +2091,6 @@ out: return true; } -#ifdef CONFIG_ACPI_CPPC_LIB -static bool amd_set_max_freq_ratio(void) -{ - struct cppc_perf_caps perf_caps; - u64 highest_perf, nominal_perf; - u64 perf_ratio; - int rc; - - rc = cppc_get_perf_caps(0, &perf_caps); - if (rc) { - pr_debug("Could not retrieve perf counters (%d)\n", rc); - return false; - } - - highest_perf = amd_get_highest_perf(); - nominal_perf = perf_caps.nominal_perf; - - if (!highest_perf || !nominal_perf) { - pr_debug("Could not retrieve highest or nominal performance\n"); - return false; - } - - perf_ratio = div_u64(highest_perf * SCHED_CAPACITY_SCALE, nominal_perf); - /* midpoint between max_boost and max_P */ - perf_ratio = (perf_ratio + SCHED_CAPACITY_SCALE) >> 1; - if (!perf_ratio) { - pr_debug("Non-zero highest/nominal perf values led to a 0 ratio\n"); - return false; - } - - arch_turbo_freq_ratio = perf_ratio; - arch_set_max_freq_ratio(false); - - return true; -} -#else -static bool amd_set_max_freq_ratio(void) -{ - return false; -} -#endif - static void init_counter_refs(void) { u64 aperf, mperf; @@ -2167,7 +2119,7 @@ static void register_freq_invariance_syscore_ops(void) static inline void register_freq_invariance_syscore_ops(void) {} #endif -static void init_freq_invariance(bool secondary, bool cppc_ready) +void init_freq_invariance(bool secondary, bool cppc_ready) { bool ret = false; @@ -2187,7 +2139,7 @@ static void init_freq_invariance(bool secondary, bool cppc_ready) if (!cppc_ready) { return; } - ret = amd_set_max_freq_ratio(); + ret = amd_set_max_freq_ratio(&arch_turbo_freq_ratio); } if (ret) { @@ -2200,22 +2152,6 @@ static void init_freq_invariance(bool secondary, bool cppc_ready) } } -#ifdef CONFIG_ACPI_CPPC_LIB -static DEFINE_MUTEX(freq_invariance_lock); - -void init_freq_invariance_cppc(void) -{ - static bool secondary; - - mutex_lock(&freq_invariance_lock); - - init_freq_invariance(secondary, true); - secondary = true; - - mutex_unlock(&freq_invariance_lock); -} -#endif - static void disable_freq_invariance_workfn(struct work_struct *work) { static_branch_disable(&arch_scale_freq_key); @@ -2264,8 +2200,4 @@ error: pr_warn("Scheduler frequency invariance went wobbly, disabling!\n"); schedule_work(&disable_freq_invariance_work); } -#else -static inline void init_freq_invariance(bool secondary, bool cppc_ready) -{ -} #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 15b058eefc4e..ee117fcf46ed 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c @@ -90,7 +90,7 @@ copy_stack_frame(const struct stack_frame_user __user *fp, { int ret; - if (__range_not_ok(fp, sizeof(*frame), TASK_SIZE)) + if (!__access_ok(fp, sizeof(*frame))) return 0; ret = 1; diff --git a/arch/x86/kernel/topology.c b/arch/x86/kernel/topology.c index bd83748e2bde..8617d1ed9d31 100644 --- a/arch/x86/kernel/topology.c +++ b/arch/x86/kernel/topology.c @@ -154,11 +154,6 @@ static int __init topology_init(void) { int i; -#ifdef CONFIG_NUMA - for_each_online_node(i) - register_one_node(i); -#endif - for_each_present_cpu(i) arch_register_cpu(i); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index c9d566dcf89a..2e37862e3a8c 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -39,6 +39,7 @@ #include <linux/io.h> #include <linux/hardirq.h> #include <linux/atomic.h> +#include <linux/ioasid.h> #include <asm/stacktrace.h> #include <asm/processor.h> @@ -559,6 +560,57 @@ static bool fixup_iopl_exception(struct pt_regs *regs) return true; } +/* + * The unprivileged ENQCMD instruction generates #GPs if the + * IA32_PASID MSR has not been populated. If possible, populate + * the MSR from a PASID previously allocated to the mm. + */ +static bool try_fixup_enqcmd_gp(void) +{ +#ifdef CONFIG_IOMMU_SVA + u32 pasid; + + /* + * MSR_IA32_PASID is managed using XSAVE. Directly + * writing to the MSR is only possible when fpregs + * are valid and the fpstate is not. This is + * guaranteed when handling a userspace exception + * in *before* interrupts are re-enabled. + */ + lockdep_assert_irqs_disabled(); + + /* + * Hardware without ENQCMD will not generate + * #GPs that can be fixed up here. + */ + if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) + return false; + + pasid = current->mm->pasid; + + /* + * If the mm has not been allocated a + * PASID, the #GP can not be fixed up. + */ + if (!pasid_valid(pasid)) + return false; + + /* + * Did this thread already have its PASID activated? + * If so, the #GP must be from something else. + */ + if (current->pasid_activated) + return false; + + wrmsrl(MSR_IA32_PASID, pasid | MSR_IA32_PASID_VALID); + current->pasid_activated = 1; + + return true; +#else + return false; +#endif +} + DEFINE_IDTENTRY_ERRORCODE(exc_general_protection) { char desc[sizeof(GPFSTR) + 50 + 2*sizeof(unsigned long) + 1] = GPFSTR; @@ -567,6 +619,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection) unsigned long gp_addr; int ret; + if (user_mode(regs) && try_fixup_enqcmd_gp()) + return; + cond_local_irq_enable(regs); if (static_cpu_has(X86_FEATURE_UMIP)) { @@ -659,6 +714,7 @@ static bool do_int3(struct pt_regs *regs) return res == NOTIFY_STOP; } +NOKPROBE_SYMBOL(do_int3); static void do_int3_user(struct pt_regs *regs) { diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 7d20c1d34a3c..e84ee5cdbd8c 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -129,6 +129,11 @@ struct x86_cpuinit_ops x86_cpuinit = { static void default_nmi_init(void) { }; +static void enc_status_change_prepare_noop(unsigned long vaddr, int npages, bool enc) { } +static bool enc_status_change_finish_noop(unsigned long vaddr, int npages, bool enc) { return false; } +static bool enc_tlb_flush_required_noop(bool enc) { return false; } +static bool enc_cache_flush_required_noop(void) { return false; } + struct x86_platform_ops x86_platform __ro_after_init = { .calibrate_cpu = native_calibrate_cpu_early, .calibrate_tsc = native_calibrate_tsc, @@ -138,9 +143,16 @@ struct x86_platform_ops x86_platform __ro_after_init = { .is_untracked_pat_range = is_ISA_range, .nmi_init = default_nmi_init, .get_nmi_reason = default_get_nmi_reason, - .save_sched_clock_state = tsc_save_sched_clock_state, - .restore_sched_clock_state = tsc_restore_sched_clock_state, + .save_sched_clock_state = tsc_save_sched_clock_state, + .restore_sched_clock_state = tsc_restore_sched_clock_state, .hyper.pin_vcpu = x86_op_int_noop, + + .guest = { + .enc_status_change_prepare = enc_status_change_prepare_noop, + .enc_status_change_finish = enc_status_change_finish_noop, + .enc_tlb_flush_required = enc_tlb_flush_required_noop, + .enc_cache_flush_required = enc_cache_flush_required_noop, + }, }; EXPORT_SYMBOL_GPL(x86_platform); diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 494d4d351859..8bf541b24b4b 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -282,6 +282,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; struct kvm_cpuid_entry2 *best; + u64 guest_supported_xcr0; best = kvm_find_cpuid_entry(vcpu, 1, 0); if (best && apic) { @@ -293,9 +294,11 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_apic_set_version(vcpu); } - vcpu->arch.guest_supported_xcr0 = + guest_supported_xcr0 = cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent); + vcpu->arch.guest_fpu.fpstate->user_xfeatures = guest_supported_xcr0; + kvm_update_pv_runtime(vcpu); vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); @@ -876,7 +879,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) eax.split.bit_width = cap.bit_width_gp; eax.split.mask_length = cap.events_mask_len; - edx.split.num_counters_fixed = min(cap.num_counters_fixed, MAX_FIXED_COUNTERS); + edx.split.num_counters_fixed = + min(cap.num_counters_fixed, KVM_PMC_MAX_FIXED); edx.split.bit_width_fixed = cap.bit_width_fixed; if (cap.version) edx.split.anythread_deprecated = 1; diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 5719d8cfdbd9..50ca1db7247c 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -429,8 +429,23 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); FOP_END /* Special case for SETcc - 1 instruction per cc */ + +/* + * Depending on .config the SETcc functions look like: + * + * SETcc %al [3 bytes] + * RET [1 byte] + * INT3 [1 byte; CONFIG_SLS] + * + * Which gives possible sizes 4 or 5. When rounded up to the + * next power-of-two alignment they become 4 or 8. + */ +#define SETCC_LENGTH (4 + IS_ENABLED(CONFIG_SLS)) +#define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS)) +static_assert(SETCC_LENGTH <= SETCC_ALIGN); + #define FOP_SETCC(op) \ - ".align 4 \n\t" \ + ".align " __stringify(SETCC_ALIGN) " \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ #op " %al \n\t" \ @@ -665,7 +680,7 @@ static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) static inline bool emul_is_noncanonical_address(u64 la, struct x86_emulate_ctxt *ctxt) { - return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; + return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt)); } /* @@ -715,7 +730,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, case X86EMUL_MODE_PROT64: *linear = la; va_bits = ctxt_virt_addr_bits(ctxt); - if (get_canonical(la, va_bits) != la) + if (!__is_canonical_address(la, va_bits)) goto bad; *max_size = min_t(u64, ~0u, (1ull << va_bits) - la); @@ -1047,7 +1062,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt) static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { u8 rc; - void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf); + void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf); flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF; asm("push %[flags]; popf; " CALL_NOSPEC @@ -5380,8 +5395,13 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) void init_decode_cache(struct x86_emulate_ctxt *ctxt) { - memset(&ctxt->rip_relative, 0, - (void *)&ctxt->modrm - (void *)&ctxt->rip_relative); + /* Clear fields that are set conditionally but read without a guard. */ + ctxt->rip_relative = false; + ctxt->rex_prefix = 0; + ctxt->lock_prefix = 0; + ctxt->rep_prefix = 0; + ctxt->regs_valid = 0; + ctxt->regs_dirty = 0; ctxt->io_read.pos = 0; ctxt->io_read.end = 0; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 39eded2426ff..840ddb4a9302 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -336,11 +336,7 @@ struct x86_emulate_ctxt { fastop_t fop; }; int (*check_perm)(struct x86_emulate_ctxt *ctxt); - /* - * The following six fields are cleared together, - * the rest are initialized unconditionally in x86_decode_insn - * or elsewhere - */ + bool rip_relative; u8 rex_prefix; u8 lock_prefix; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index d7e6fde82d25..9322e6340a74 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2306,7 +2306,12 @@ void kvm_apic_update_apicv(struct kvm_vcpu *vcpu) apic->irr_pending = true; apic->isr_count = 1; } else { - apic->irr_pending = (apic_search_irr(apic) != -1); + /* + * Don't clear irr_pending, searching the IRR can race with + * updates from the CPU as APICv is still active from hardware's + * perspective. The flag will be cleared as appropriate when + * KVM injects the interrupt. + */ apic->isr_count = count_vectors(apic->regs + APIC_ISR); } } diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 593093b52395..5628d0ba637e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3565,7 +3565,7 @@ set_root_pgd: out_unlock: write_unlock(&vcpu->kvm->mmu_lock); - return 0; + return r; } static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) @@ -3889,12 +3889,23 @@ static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr) walk_shadow_page_lockless_end(vcpu); } +static u32 alloc_apf_token(struct kvm_vcpu *vcpu) +{ + /* make sure the token value is not 0 */ + u32 id = vcpu->arch.apf.id; + + if (id << 12 == 0) + vcpu->arch.apf.id = 1; + + return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; +} + static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, gfn_t gfn) { struct kvm_arch_async_pf arch; - arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; + arch.token = alloc_apf_token(vcpu); arch.gfn = gfn; arch.direct_map = vcpu->arch.mmu->direct_map; arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index f614f95acc6b..b1a02993782b 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -95,7 +95,7 @@ static void kvm_perf_overflow(struct perf_event *perf_event, } static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, - unsigned config, bool exclude_user, + u64 config, bool exclude_user, bool exclude_kernel, bool intr, bool in_tx, bool in_tx_cp) { @@ -181,7 +181,8 @@ static int cmp_u64(const void *a, const void *b) void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) { - unsigned config, type = PERF_TYPE_RAW; + u64 config; + u32 type = PERF_TYPE_RAW; struct kvm *kvm = pmc->vcpu->kvm; struct kvm_pmu_event_filter *filter; bool allow_event = true; @@ -220,7 +221,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) } if (type == PERF_TYPE_RAW) - config = eventsel & X86_RAW_EVENT_MASK; + config = eventsel & AMD64_RAW_EVENT_MASK; if (pmc->current_config == eventsel && pmc_resume_counter(pmc)) return; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 7a7b8d5b775e..9e66fba1d6a3 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -15,8 +15,6 @@ #define VMWARE_BACKDOOR_PMC_REAL_TIME 0x10001 #define VMWARE_BACKDOOR_PMC_APPARENT_TIME 0x10002 -#define MAX_FIXED_COUNTERS 3 - struct kvm_event_hw_type_mapping { u8 eventsel; u8 unit_mask; diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 90364d02f22a..fb3e20791338 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -27,20 +27,6 @@ #include "irq.h" #include "svm.h" -#define SVM_AVIC_DOORBELL 0xc001011b - -#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) - -/* - * 0xff is broadcast, so the max index allowed for physical APIC ID - * table is 0xfe. APIC IDs above 0xff are reserved. - */ -#define AVIC_MAX_PHYSICAL_ID_COUNT 255 - -#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 -#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 -#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF - /* AVIC GATAG is encoded using VM and VCPU IDs */ #define AVIC_VCPU_ID_BITS 8 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) @@ -73,12 +59,6 @@ struct amd_svm_iommu_ir { void *data; /* Storing pointer to struct amd_ir_data */ }; -enum avic_ipi_failure_cause { - AVIC_IPI_FAILURE_INVALID_INT_TYPE, - AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, - AVIC_IPI_FAILURE_INVALID_TARGET, - AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, -}; /* Note: * This function is called from IOMMU driver to notify @@ -289,6 +269,22 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu) return 0; } +void avic_ring_doorbell(struct kvm_vcpu *vcpu) +{ + /* + * Note, the vCPU could get migrated to a different pCPU at any point, + * which could result in signalling the wrong/previous pCPU. But if + * that happens the vCPU is guaranteed to do a VMRUN (after being + * migrated) and thus will process pending interrupts, i.e. a doorbell + * is not needed (and the spurious one is harmless). + */ + int cpu = READ_ONCE(vcpu->cpu); + + if (cpu != get_cpu()) + wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); + put_cpu(); +} + static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, u32 icrl, u32 icrh) { @@ -304,8 +300,13 @@ static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, kvm_for_each_vcpu(i, vcpu, kvm) { if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK, GET_APIC_DEST_FIELD(icrh), - icrl & APIC_DEST_MASK)) - kvm_vcpu_wake_up(vcpu); + icrl & APIC_DEST_MASK)) { + vcpu->arch.apic->irr_pending = true; + svm_complete_interrupt_delivery(vcpu, + icrl & APIC_MODE_MASK, + icrl & APIC_INT_LEVELTRIG, + icrl & APIC_VECTOR_MASK); + } } } @@ -345,8 +346,6 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh); break; case AVIC_IPI_FAILURE_INVALID_TARGET: - WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", - index, vcpu->vcpu_id, icrh, icrl); break; case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: WARN_ONCE(1, "Invalid backing page\n"); @@ -669,52 +668,6 @@ void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) return; } -int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) -{ - if (!vcpu->arch.apicv_active) - return -1; - - kvm_lapic_set_irr(vec, vcpu->arch.apic); - - /* - * Pairs with the smp_mb_*() after setting vcpu->guest_mode in - * vcpu_enter_guest() to ensure the write to the vIRR is ordered before - * the read of guest_mode, which guarantees that either VMRUN will see - * and process the new vIRR entry, or that the below code will signal - * the doorbell if the vCPU is already running in the guest. - */ - smp_mb__after_atomic(); - - /* - * Signal the doorbell to tell hardware to inject the IRQ if the vCPU - * is in the guest. If the vCPU is not in the guest, hardware will - * automatically process AVIC interrupts at VMRUN. - */ - if (vcpu->mode == IN_GUEST_MODE) { - int cpu = READ_ONCE(vcpu->cpu); - - /* - * Note, the vCPU could get migrated to a different pCPU at any - * point, which could result in signalling the wrong/previous - * pCPU. But if that happens the vCPU is guaranteed to do a - * VMRUN (after being migrated) and thus will process pending - * interrupts, i.e. a doorbell is not needed (and the spurious - * one is harmless). - */ - if (cpu != get_cpu()) - wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); - put_cpu(); - } else { - /* - * Wake the vCPU if it was blocking. KVM will then detect the - * pending IRQ when checking if the vCPU has a wake event. - */ - kvm_vcpu_wake_up(vcpu); - } - - return 0; -} - bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) { return false; diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1218b5a342fc..39d280e7e80e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1457,18 +1457,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, !__nested_vmcb_check_save(vcpu, &save_cached)) goto out_free; - /* - * While the nested guest CR3 is already checked and set by - * KVM_SET_SREGS, it was set when nested state was yet loaded, - * thus MMU might not be initialized correctly. - * Set it again to fix this. - */ - - ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, - nested_npt_enabled(svm), false); - if (WARN_ON_ONCE(ret)) - goto out_free; - /* * All checks done, we can enter guest mode. Userspace provides @@ -1494,6 +1482,20 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm_switch_vmcb(svm, &svm->nested.vmcb02); nested_vmcb02_prepare_control(svm); + + /* + * While the nested guest CR3 is already checked and set by + * KVM_SET_SREGS, it was set when nested state was yet loaded, + * thus MMU might not be initialized correctly. + * Set it again to fix this. + */ + + ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, + nested_npt_enabled(svm), false); + if (WARN_ON_ONCE(ret)) + goto out_free; + + kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); ret = 0; out_free: diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a290efb272ad..fd3a00c892c7 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1585,6 +1585,7 @@ void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); u64 hcr0 = cr0; + bool old_paging = is_paging(vcpu); #ifdef CONFIG_X86_64 if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) { @@ -1601,8 +1602,11 @@ void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) #endif vcpu->arch.cr0 = cr0; - if (!npt_enabled) + if (!npt_enabled) { hcr0 |= X86_CR0_PG | X86_CR0_WP; + if (old_paging != is_paging(vcpu)) + svm_set_cr4(vcpu, kvm_read_cr4(vcpu)); + } /* * re-enable caching here because the QEMU bios @@ -1646,8 +1650,12 @@ void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) svm_flush_tlb(vcpu); vcpu->arch.cr4 = cr4; - if (!npt_enabled) + if (!npt_enabled) { cr4 |= X86_CR4_PAE; + + if (!is_paging(vcpu)) + cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); + } cr4 |= host_cr4_mce; to_svm(vcpu)->vmcb->save.cr4 = cr4; vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); @@ -2685,8 +2693,23 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) u64 data = msr->data; switch (ecx) { case MSR_AMD64_TSC_RATIO: - if (!msr->host_initiated && !svm->tsc_scaling_enabled) - return 1; + + if (!svm->tsc_scaling_enabled) { + + if (!msr->host_initiated) + return 1; + /* + * In case TSC scaling is not enabled, always + * leave this MSR at the default value. + * + * Due to bug in qemu 6.2.0, it would try to set + * this msr to 0 if tsc scaling is not enabled. + * Ignore this value as well. + */ + if (data != 0 && data != svm->tsc_ratio_msr) + return 1; + break; + } if (data & TSC_RATIO_RSVD) return 1; @@ -3291,21 +3314,55 @@ static void svm_set_irq(struct kvm_vcpu *vcpu) SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; } -static void svm_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, - int trig_mode, int vector) +void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, + int trig_mode, int vector) { - struct kvm_vcpu *vcpu = apic->vcpu; + /* + * vcpu->arch.apicv_active must be read after vcpu->mode. + * Pairs with smp_store_release in vcpu_enter_guest. + */ + bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); - if (svm_deliver_avic_intr(vcpu, vector)) { - kvm_lapic_set_irr(vector, apic); + if (!READ_ONCE(vcpu->arch.apicv_active)) { + /* Process the interrupt via inject_pending_event */ kvm_make_request(KVM_REQ_EVENT, vcpu); kvm_vcpu_kick(vcpu); + return; + } + + trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); + if (in_guest_mode) { + /* + * Signal the doorbell to tell hardware to inject the IRQ. If + * the vCPU exits the guest before the doorbell chimes, hardware + * will automatically process AVIC interrupts at the next VMRUN. + */ + avic_ring_doorbell(vcpu); } else { - trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, - trig_mode, vector); + /* + * Wake the vCPU if it was blocking. KVM will then detect the + * pending IRQ when checking if the vCPU has a wake event. + */ + kvm_vcpu_wake_up(vcpu); } } +static void svm_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, + int trig_mode, int vector) +{ + kvm_lapic_set_irr(vector, apic); + + /* + * Pairs with the smp_mb_*() after setting vcpu->guest_mode in + * vcpu_enter_guest() to ensure the write to the vIRR is ordered before + * the read of guest_mode. This guarantees that either VMRUN will see + * and process the new vIRR entry, or that svm_complete_interrupt_delivery + * will signal the doorbell if the CPU has already entered the guest. + */ + smp_mb__after_atomic(); + svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); +} + static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3353,11 +3410,13 @@ static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_nmi_blocked(vcpu)) + return 0; + /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) return -EBUSY; - - return !svm_nmi_blocked(vcpu); + return 1; } static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) @@ -3409,9 +3468,13 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); + if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_interrupt_blocked(vcpu)) + return 0; + /* * An IRQ must not be injected into L2 if it's supposed to VM-Exit, * e.g. if the IRQ arrived asynchronously after checking nested events. @@ -3419,7 +3482,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) return -EBUSY; - return !svm_interrupt_blocked(vcpu); + return 1; } static void svm_enable_irq_window(struct kvm_vcpu *vcpu) @@ -4150,11 +4213,14 @@ static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_smi_blocked(vcpu)) + return 0; + /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) return -EBUSY; - return !svm_smi_blocked(vcpu); + return 1; } static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) @@ -4248,11 +4314,18 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) * Enter the nested guest now */ + vmcb_mark_all_dirty(svm->vmcb01.ptr); + vmcb12 = map.hva; nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false); + if (ret) + goto unmap_save; + + svm->nested.nested_run_pending = 1; + unmap_save: kvm_vcpu_unmap(vcpu, &map_save, true); unmap_map: @@ -4637,6 +4710,7 @@ static __init void svm_set_cpu_caps(void) /* CPUID 0x80000001 and 0x8000000A (SVM features) */ if (nested) { kvm_cpu_cap_set(X86_FEATURE_SVM); + kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN); if (nrips) kvm_cpu_cap_set(X86_FEATURE_NRIPS); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 73525353e424..fa98d6844728 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -489,6 +489,8 @@ void svm_set_gif(struct vcpu_svm *svm, bool value); int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code); void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, int read, int write); +void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, + int trig_mode, int vec); /* nested.c */ @@ -556,17 +558,6 @@ extern struct kvm_x86_nested_ops svm_nested_ops; /* avic.c */ -#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) -#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 -#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) - -#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) -#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) -#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) -#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) - -#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL - int avic_ga_log_notifier(u32 ga_tag); void avic_vm_destroy(struct kvm *kvm); int avic_vm_init(struct kvm *kvm); @@ -583,12 +574,12 @@ bool svm_check_apicv_inhibit_reasons(ulong bit); void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr); void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr); -int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec); bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu); int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, bool set); void avic_vcpu_blocking(struct kvm_vcpu *vcpu); void avic_vcpu_unblocking(struct kvm_vcpu *vcpu); +void avic_ring_doorbell(struct kvm_vcpu *vcpu); /* sev.c */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ba34e94049c7..dc822a1d403d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -246,8 +246,7 @@ static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, src = &prev->host_state; dest = &vmx->loaded_vmcs->host_state; - vmx_set_vmcs_host_state(dest, src->cr3, src->fs_sel, src->gs_sel, - src->fs_base, src->gs_base); + vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); dest->ldt_sel = src->ldt_sel; #ifdef CONFIG_X86_64 dest->ds_sel = src->ds_sel; @@ -3056,7 +3055,7 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - unsigned long cr4; + unsigned long cr3, cr4; bool vm_fail; if (!nested_early_check) @@ -3079,6 +3078,12 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) */ vmcs_writel(GUEST_RFLAGS, 0); + cr3 = __get_current_cr3_fast(); + if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { + vmcs_writel(HOST_CR3, cr3); + vmx->loaded_vmcs->host_state.cr3 = cr3; + } + cr4 = cr4_read_shadow(); if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { vmcs_writel(HOST_CR4, cr4); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 466d18fc0c5d..9b26596099a1 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -565,7 +565,7 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) pmu->gp_counters[i].current_config = 0; } - for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { + for (i = 0; i < KVM_PMC_MAX_FIXED; i++) { pmu->fixed_counters[i].type = KVM_PMC_FIXED; pmu->fixed_counters[i].vcpu = vcpu; pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED; @@ -591,7 +591,7 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) pmc->counter = pmc->eventsel = 0; } - for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { + for (i = 0; i < KVM_PMC_MAX_FIXED; i++) { pmc = &pmu->fixed_counters[i]; pmc_stop_counter(pmc); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 6c27bd0c89e1..b730d799c26e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1080,14 +1080,9 @@ static void pt_guest_exit(struct vcpu_vmx *vmx) wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); } -void vmx_set_vmcs_host_state(struct vmcs_host_state *host, unsigned long cr3, - u16 fs_sel, u16 gs_sel, - unsigned long fs_base, unsigned long gs_base) +void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, + unsigned long fs_base, unsigned long gs_base) { - if (unlikely(cr3 != host->cr3)) { - vmcs_writel(HOST_CR3, cr3); - host->cr3 = cr3; - } if (unlikely(fs_sel != host->fs_sel)) { if (!(fs_sel & 7)) vmcs_write16(HOST_FS_SELECTOR, fs_sel); @@ -1182,9 +1177,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) gs_base = segment_base(gs_sel); #endif - vmx_set_vmcs_host_state(host_state, __get_current_cr3_fast(), - fs_sel, gs_sel, fs_base, gs_base); - + vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); vmx->guest_state_loaded = true; } @@ -6791,7 +6784,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - unsigned long cr4; + unsigned long cr3, cr4; /* Record the guest's net vcpu time for enforced NMI injections. */ if (unlikely(!enable_vnmi && @@ -6834,6 +6827,19 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); vcpu->arch.regs_dirty = 0; + /* + * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately + * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time + * it switches back to the current->mm, which can occur in KVM context + * when switching to a temporary mm to patch kernel code, e.g. if KVM + * toggles a static key while handling a VM-Exit. + */ + cr3 = __get_current_cr3_fast(); + if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { + vmcs_writel(HOST_CR3, cr3); + vmx->loaded_vmcs->host_state.cr3 = cr3; + } + cr4 = cr4_read_shadow(); if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { vmcs_writel(HOST_CR4, cr4); @@ -7659,6 +7665,7 @@ static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) if (ret) return ret; + vmx->nested.nested_run_pending = 1; vmx->nested.smm.guest_mode = false; } return 0; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 7f2c82e7f38f..9c6bfcd84008 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -374,9 +374,8 @@ int allocate_vpid(void); void free_vpid(int vpid); void vmx_set_constant_host_state(struct vcpu_vmx *vmx); void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu); -void vmx_set_vmcs_host_state(struct vmcs_host_state *host, unsigned long cr3, - u16 fs_sel, u16 gs_sel, - unsigned long fs_base, unsigned long gs_base); +void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, + unsigned long fs_base, unsigned long gs_base); int vmx_get_cpl(struct kvm_vcpu *vcpu); bool vmx_emulation_required(struct kvm_vcpu *vcpu); unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7131d735b1ef..6db3a506b402 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -984,6 +984,18 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state); +static inline u64 kvm_guest_supported_xcr0(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->user_xfeatures; +} + +#ifdef CONFIG_X86_64 +static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) +{ + return kvm_guest_supported_xcr0(vcpu) & XFEATURE_MASK_USER_DYNAMIC; +} +#endif + static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) { u64 xcr0 = xcr; @@ -1003,7 +1015,7 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) * saving. However, xcr0 bit 0 is always set, even if the * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). */ - valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; + valid_bits = kvm_guest_supported_xcr0(vcpu) | XFEATURE_MASK_FP; if (xcr0 & ~valid_bits) return 1; @@ -1737,7 +1749,7 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, * value, and that something deterministic happens if the guest * invokes 64-bit SYSENTER. */ - data = get_canonical(data, vcpu_virt_addr_bits(vcpu)); + data = __canonical_address(data, vcpu_virt_addr_bits(vcpu)); break; case MSR_TSC_AUX: if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) @@ -2351,10 +2363,12 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) return tsc; } +#ifdef CONFIG_X86_64 static inline int gtod_is_based_on_tsc(int mode) { return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; } +#endif static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) { @@ -3706,8 +3720,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) return 1; - if (data & ~(XFEATURE_MASK_USER_DYNAMIC & - vcpu->arch.guest_supported_xcr0)) + if (data & ~kvm_guest_supported_xfd(vcpu)) return 1; fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); @@ -3717,8 +3730,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) return 1; - if (data & ~(XFEATURE_MASK_USER_DYNAMIC & - vcpu->arch.guest_supported_xcr0)) + if (data & ~kvm_guest_supported_xfd(vcpu)) return 1; vcpu->arch.guest_fpu.xfd_err = data; @@ -4233,6 +4245,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_EXIT_ON_EMULATION_FAILURE: case KVM_CAP_VCPU_ATTRIBUTES: case KVM_CAP_SYS_ATTRIBUTES: + case KVM_CAP_ENABLE_CAP: r = 1; break; case KVM_CAP_EXIT_HYPERCALL: @@ -6516,7 +6529,7 @@ static void kvm_init_msr_list(void) u32 dummy[2]; unsigned i; - BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4, + BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3, "Please update the fixed PMCs in msrs_to_saved_all[]"); perf_get_x86_pmu_capability(&x86_pmu); @@ -8840,7 +8853,7 @@ int kvm_arch_init(void *opaque) } if (pi_inject_timer == -1) - pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER); + pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER); #ifdef CONFIG_X86_64 pvclock_gtod_register_notifier(&pvclock_gtod_notifier); @@ -8942,6 +8955,13 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK) return -KVM_EOPNOTSUPP; + /* + * When tsc is in permanent catchup mode guests won't be able to use + * pvclock_read_retry loop to get consistent view of pvclock + */ + if (vcpu->arch.tsc_always_catchup) + return -KVM_EOPNOTSUPP; + if (!kvm_get_walltime_and_clockread(&ts, &cycle)) return -KVM_EOPNOTSUPP; @@ -9160,6 +9180,7 @@ static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) likely(!pic_in_kernel(vcpu->kvm)); } +/* Called within kvm->srcu read side. */ static void post_kvm_run_save(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; @@ -9168,16 +9189,9 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) kvm_run->cr8 = kvm_get_cr8(vcpu); kvm_run->apic_base = kvm_get_apic_base(vcpu); - /* - * The call to kvm_ready_for_interrupt_injection() may end up in - * kvm_xen_has_interrupt() which may require the srcu lock to be - * held, to protect against changes in the vcpu_info address. - */ - vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); kvm_run->ready_for_interrupt_injection = pic_in_kernel(vcpu->kvm) || kvm_vcpu_ready_for_interrupt_injection(vcpu); - srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); if (is_smm(vcpu)) kvm_run->flags |= KVM_RUN_X86_SMM; @@ -9795,6 +9809,7 @@ void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu) EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit); /* + * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without * exiting to the userspace. Otherwise, the value will be returned to the * userspace. @@ -9983,7 +9998,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * result in virtual interrupt delivery. */ local_irq_disable(); - vcpu->mode = IN_GUEST_MODE; + + /* Store vcpu->apicv_active before vcpu->mode. */ + smp_store_release(&vcpu->mode, IN_GUEST_MODE); srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); @@ -10171,6 +10188,7 @@ out: return r; } +/* Called within kvm->srcu read side. */ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) { bool hv_timer; @@ -10230,12 +10248,12 @@ static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) !vcpu->arch.apf.halted); } +/* Called within kvm->srcu read side. */ static int vcpu_run(struct kvm_vcpu *vcpu) { int r; struct kvm *kvm = vcpu->kvm; - vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); vcpu->arch.l1tf_flush_l1d = true; for (;;) { @@ -10263,14 +10281,12 @@ static int vcpu_run(struct kvm_vcpu *vcpu) if (__xfer_to_guest_mode_work_pending()) { srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); r = xfer_to_guest_mode_handle_work(vcpu); + vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); if (r) return r; - vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); } } - srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); - return r; } @@ -10376,6 +10392,7 @@ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; + struct kvm *kvm = vcpu->kvm; int r; vcpu_load(vcpu); @@ -10383,6 +10400,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_run->flags = 0; kvm_load_guest_fpu(vcpu); + vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { if (kvm_run->immediate_exit) { r = -EINTR; @@ -10393,7 +10411,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) * use before KVM has ever run the vCPU. */ WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu)); + + srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); kvm_vcpu_block(vcpu); + vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); + if (kvm_apic_accept_events(vcpu) < 0) { r = 0; goto out; @@ -10453,8 +10475,9 @@ out: if (kvm_run->kvm_valid_regs) store_regs(vcpu); post_kvm_run_save(vcpu); - kvm_sigset_deactivate(vcpu); + srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); + kvm_sigset_deactivate(vcpu); vcpu_put(vcpu); return r; } diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 767ec7f99516..f11d945ac41f 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -166,14 +166,9 @@ static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu) return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48; } -static inline u64 get_canonical(u64 la, u8 vaddr_bits) -{ - return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits); -} - static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu) { - return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la; + return !__is_canonical_address(la, vcpu_virt_addr_bits(vcpu)); } static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index bad57535fad0..74be1fda58e3 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -133,32 +133,57 @@ static void kvm_xen_update_runstate(struct kvm_vcpu *v, int state) void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) { struct kvm_vcpu_xen *vx = &v->arch.xen; + struct gfn_to_hva_cache *ghc = &vx->runstate_cache; + struct kvm_memslots *slots = kvm_memslots(v->kvm); + bool atomic = (state == RUNSTATE_runnable); uint64_t state_entry_time; - unsigned int offset; + int __user *user_state; + uint64_t __user *user_times; kvm_xen_update_runstate(v, state); if (!vx->runstate_set) return; - BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c); + if (unlikely(slots->generation != ghc->generation || kvm_is_error_hva(ghc->hva)) && + kvm_gfn_to_hva_cache_init(v->kvm, ghc, ghc->gpa, ghc->len)) + return; + + /* We made sure it fits in a single page */ + BUG_ON(!ghc->memslot); + + if (atomic) + pagefault_disable(); - offset = offsetof(struct compat_vcpu_runstate_info, state_entry_time); -#ifdef CONFIG_X86_64 /* - * The only difference is alignment of uint64_t in 32-bit. - * So the first field 'state' is accessed directly using - * offsetof() (where its offset happens to be zero), while the - * remaining fields which are all uint64_t, start at 'offset' - * which we tweak here by adding 4. + * The only difference between 32-bit and 64-bit versions of the + * runstate struct us the alignment of uint64_t in 32-bit, which + * means that the 64-bit version has an additional 4 bytes of + * padding after the first field 'state'. + * + * So we use 'int __user *user_state' to point to the state field, + * and 'uint64_t __user *user_times' for runstate_entry_time. So + * the actual array of time[] in each state starts at user_times[1]. */ + BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != 0); + BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state) != 0); + user_state = (int __user *)ghc->hva; + + BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c); + + user_times = (uint64_t __user *)(ghc->hva + + offsetof(struct compat_vcpu_runstate_info, + state_entry_time)); +#ifdef CONFIG_X86_64 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) != offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4); BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) != offsetof(struct compat_vcpu_runstate_info, time) + 4); if (v->kvm->arch.xen.long_mode) - offset = offsetof(struct vcpu_runstate_info, state_entry_time); + user_times = (uint64_t __user *)(ghc->hva + + offsetof(struct vcpu_runstate_info, + state_entry_time)); #endif /* * First write the updated state_entry_time at the appropriate @@ -172,10 +197,8 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) != sizeof(state_entry_time)); - if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache, - &state_entry_time, offset, - sizeof(state_entry_time))) - return; + if (__put_user(state_entry_time, user_times)) + goto out; smp_wmb(); /* @@ -189,11 +212,8 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) != sizeof(vx->current_runstate)); - if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache, - &vx->current_runstate, - offsetof(struct vcpu_runstate_info, state), - sizeof(vx->current_runstate))) - return; + if (__put_user(vx->current_runstate, user_state)) + goto out; /* * Write the actual runstate times immediately after the @@ -208,24 +228,23 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache, - &vx->runstate_times[0], - offset + sizeof(u64), - sizeof(vx->runstate_times))) - return; - + if (__copy_to_user(user_times + 1, vx->runstate_times, sizeof(vx->runstate_times))) + goto out; smp_wmb(); /* * Finally, clear the XEN_RUNSTATE_UPDATE bit in the guest's * runstate_entry_time field. */ - state_entry_time &= ~XEN_RUNSTATE_UPDATE; - if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache, - &state_entry_time, offset, - sizeof(state_entry_time))) - return; + __put_user(state_entry_time, user_times); + smp_wmb(); + + out: + mark_page_dirty_in_slot(v->kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); + + if (atomic) + pagefault_enable(); } int __kvm_xen_has_interrupt(struct kvm_vcpu *v) @@ -443,6 +462,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + /* It must fit within a single page */ + if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_info) > PAGE_SIZE) { + r = -EINVAL; + break; + } + r = kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache, data->u.gpa, @@ -460,6 +485,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + /* It must fit within a single page */ + if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct pvclock_vcpu_time_info) > PAGE_SIZE) { + r = -EINVAL; + break; + } + r = kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.xen.vcpu_time_info_cache, data->u.gpa, @@ -481,6 +512,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + /* It must fit within a single page */ + if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_runstate_info) > PAGE_SIZE) { + r = -EINVAL; + break; + } + r = kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.xen.runstate_cache, data->u.gpa, diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 59cf2343f3d9..d0d7b9bc6cad 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -27,8 +27,7 @@ * Output: * rax original destination */ -SYM_FUNC_START_ALIAS(__memcpy) -SYM_FUNC_START_WEAK(memcpy) +SYM_FUNC_START(__memcpy) ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ "jmp memcpy_erms", X86_FEATURE_ERMS @@ -40,11 +39,12 @@ SYM_FUNC_START_WEAK(memcpy) movl %edx, %ecx rep movsb RET -SYM_FUNC_END(memcpy) -SYM_FUNC_END_ALIAS(__memcpy) -EXPORT_SYMBOL(memcpy) +SYM_FUNC_END(__memcpy) EXPORT_SYMBOL(__memcpy) +SYM_FUNC_ALIAS_WEAK(memcpy, __memcpy) +EXPORT_SYMBOL(memcpy) + /* * memcpy_erms() - enhanced fast string memcpy. This is faster and * simpler than memcpy. Use memcpy_erms when possible. diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index 50ea390df712..d83cba364e31 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S @@ -24,7 +24,6 @@ * Output: * rax: dest */ -SYM_FUNC_START_WEAK(memmove) SYM_FUNC_START(__memmove) mov %rdi, %rax @@ -207,6 +206,7 @@ SYM_FUNC_START(__memmove) 13: RET SYM_FUNC_END(__memmove) -SYM_FUNC_END_ALIAS(memmove) EXPORT_SYMBOL(__memmove) + +SYM_FUNC_ALIAS_WEAK(memmove, __memmove) EXPORT_SYMBOL(memmove) diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index d624f2bc42f1..fc9ffd3ff3b2 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S @@ -17,7 +17,6 @@ * * rax original destination */ -SYM_FUNC_START_WEAK(memset) SYM_FUNC_START(__memset) /* * Some CPUs support enhanced REP MOVSB/STOSB feature. It is recommended @@ -42,10 +41,11 @@ SYM_FUNC_START(__memset) movq %r9,%rax RET SYM_FUNC_END(__memset) -SYM_FUNC_END_ALIAS(memset) -EXPORT_SYMBOL(memset) EXPORT_SYMBOL(__memset) +SYM_FUNC_ALIAS_WEAK(memset, __memset) +EXPORT_SYMBOL(memset) + /* * ISO C memset - set a memory block to a byte value. This function uses * enhanced rep stosb to override the fast string function. diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 89b3fb244e15..afbdda539b80 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE .endm diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index c3e8a62ca561..ad0139d25401 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -32,7 +32,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) { unsigned long ret; - if (__range_not_ok(from, n, TASK_SIZE)) + if (!__access_ok(from, n)) return n; if (!nmi_uaccess_okay()) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index ec31f5b60323..d12d1358f96d 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -690,7 +690,10 @@ AVXcode: 2 45: vpsrlvd/q Vx,Hx,Wx (66),(v) 46: vpsravd Vx,Hx,Wx (66),(v) | vpsravd/q Vx,Hx,Wx (66),(evo) 47: vpsllvd/q Vx,Hx,Wx (66),(v) -# Skip 0x48-0x4b +# Skip 0x48 +49: TILERELEASE (v1),(000),(11B) | LDTILECFG Mtc (v1)(000) | STTILECFG Mtc (66),(v1),(000) | TILEZERO Vt (F2),(v1),(11B) +# Skip 0x4a +4b: TILELOADD Vt,Wsm (F2),(v1) | TILELOADDT1 Vt,Wsm (66),(v1) | TILESTORED Wsm,Vt (F3),(v) 4c: vrcp14ps/d Vpd,Wpd (66),(ev) 4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) 4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) @@ -705,7 +708,10 @@ AVXcode: 2 59: vpbroadcastq Vx,Wx (66),(v) | vbroadcasti32x2 Vx,Wx (66),(evo) 5a: vbroadcasti128 Vqq,Mdq (66),(v) | vbroadcasti32x4/64x2 Vx,Wx (66),(evo) 5b: vbroadcasti32x8/64x4 Vqq,Mdq (66),(ev) -# Skip 0x5c-0x61 +5c: TDPBF16PS Vt,Wt,Ht (F3),(v1) +# Skip 0x5d +5e: TDPBSSD Vt,Wt,Ht (F2),(v1) | TDPBSUD Vt,Wt,Ht (F3),(v1) | TDPBUSD Vt,Wt,Ht (66),(v1) | TDPBUUD Vt,Wt,Ht (v1) +# Skip 0x5f-0x61 62: vpexpandb/w Vx,Wx (66),(ev) 63: vpcompressb/w Wx,Vx (66),(ev) 64: vpblendmd/q Vx,Hx,Wx (66),(ev) @@ -822,9 +828,9 @@ AVXcode: 3 05: vpermilpd Vx,Wx,Ib (66),(v) 06: vperm2f128 Vqq,Hqq,Wqq,Ib (66),(v) 07: -08: vroundps Vx,Wx,Ib (66) | vrndscaleps Vx,Wx,Ib (66),(evo) +08: vroundps Vx,Wx,Ib (66) | vrndscaleps Vx,Wx,Ib (66),(evo) | vrndscaleph Vx,Wx,Ib (evo) 09: vroundpd Vx,Wx,Ib (66) | vrndscalepd Vx,Wx,Ib (66),(evo) -0a: vroundss Vss,Wss,Ib (66),(v1) | vrndscaless Vx,Hx,Wx,Ib (66),(evo) +0a: vroundss Vss,Wss,Ib (66),(v1) | vrndscaless Vx,Hx,Wx,Ib (66),(evo) | vrndscalesh Vx,Hx,Wx,Ib (evo) 0b: vroundsd Vsd,Wsd,Ib (66),(v1) | vrndscalesd Vx,Hx,Wx,Ib (66),(evo) 0c: vblendps Vx,Hx,Wx,Ib (66) 0d: vblendpd Vx,Hx,Wx,Ib (66) @@ -846,8 +852,8 @@ AVXcode: 3 22: vpinsrd/q Vdq,Hdq,Ey,Ib (66),(v1) 23: vshuff32x4/64x2 Vx,Hx,Wx,Ib (66),(ev) 25: vpternlogd/q Vx,Hx,Wx,Ib (66),(ev) -26: vgetmantps/d Vx,Wx,Ib (66),(ev) -27: vgetmantss/d Vx,Hx,Wx,Ib (66),(ev) +26: vgetmantps/d Vx,Wx,Ib (66),(ev) | vgetmantph Vx,Wx,Ib (ev) +27: vgetmantss/d Vx,Hx,Wx,Ib (66),(ev) | vgetmantsh Vx,Hx,Wx,Ib (ev) 30: kshiftrb/w Vk,Uk,Ib (66),(v) 31: kshiftrd/q Vk,Uk,Ib (66),(v) 32: kshiftlb/w Vk,Uk,Ib (66),(v) @@ -871,23 +877,102 @@ AVXcode: 3 51: vrangess/d Vx,Hx,Wx,Ib (66),(ev) 54: vfixupimmps/d Vx,Hx,Wx,Ib (66),(ev) 55: vfixupimmss/d Vx,Hx,Wx,Ib (66),(ev) -56: vreduceps/d Vx,Wx,Ib (66),(ev) -57: vreducess/d Vx,Hx,Wx,Ib (66),(ev) +56: vreduceps/d Vx,Wx,Ib (66),(ev) | vreduceph Vx,Wx,Ib (ev) +57: vreducess/d Vx,Hx,Wx,Ib (66),(ev) | vreducesh Vx,Hx,Wx,Ib (ev) 60: vpcmpestrm Vdq,Wdq,Ib (66),(v1) 61: vpcmpestri Vdq,Wdq,Ib (66),(v1) 62: vpcmpistrm Vdq,Wdq,Ib (66),(v1) 63: vpcmpistri Vdq,Wdq,Ib (66),(v1) -66: vfpclassps/d Vk,Wx,Ib (66),(ev) -67: vfpclassss/d Vk,Wx,Ib (66),(ev) +66: vfpclassps/d Vk,Wx,Ib (66),(ev) | vfpclassph Vx,Wx,Ib (ev) +67: vfpclassss/d Vk,Wx,Ib (66),(ev) | vfpclasssh Vx,Wx,Ib (ev) 70: vpshldw Vx,Hx,Wx,Ib (66),(ev) 71: vpshldd/q Vx,Hx,Wx,Ib (66),(ev) 72: vpshrdw Vx,Hx,Wx,Ib (66),(ev) 73: vpshrdd/q Vx,Hx,Wx,Ib (66),(ev) +c2: vcmpph Vx,Hx,Wx,Ib (ev) | vcmpsh Vx,Hx,Wx,Ib (F3),(ev) cc: sha1rnds4 Vdq,Wdq,Ib ce: vgf2p8affineqb Vx,Wx,Ib (66) cf: vgf2p8affineinvqb Vx,Wx,Ib (66) df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) -f0: RORX Gy,Ey,Ib (F2),(v) +f0: RORX Gy,Ey,Ib (F2),(v) | HRESET Gv,Ib (F3),(000),(11B) +EndTable + +Table: EVEX map 5 +Referrer: +AVXcode: 5 +10: vmovsh Vx,Hx,Wx (F3),(ev) | vmovsh Vx,Wx (F3),(ev) +11: vmovsh Wx,Hx,Vx (F3),(ev) | vmovsh Wx,Vx (F3),(ev) +1d: vcvtps2phx Vx,Wx (66),(ev) | vcvtss2sh Vx,Hx,Wx (ev) +2a: vcvtsi2sh Vx,Hx,Wx (F3),(ev) +2c: vcvttsh2si Vx,Wx (F3),(ev) +2d: vcvtsh2si Vx,Wx (F3),(ev) +2e: vucomish Vx,Wx (ev) +2f: vcomish Vx,Wx (ev) +51: vsqrtph Vx,Wx (ev) | vsqrtsh Vx,Hx,Wx (F3),(ev) +58: vaddph Vx,Hx,Wx (ev) | vaddsh Vx,Hx,Wx (F3),(ev) +59: vmulph Vx,Hx,Wx (ev) | vmulsh Vx,Hx,Wx (F3),(ev) +5a: vcvtpd2ph Vx,Wx (66),(ev) | vcvtph2pd Vx,Wx (ev) | vcvtsd2sh Vx,Hx,Wx (F2),(ev) | vcvtsh2sd Vx,Hx,Wx (F3),(ev) +5b: vcvtdq2ph Vx,Wx (ev) | vcvtph2dq Vx,Wx (66),(ev) | vcvtqq2ph Vx,Wx (ev) | vcvttph2dq Vx,Wx (F3),(ev) +5c: vsubph Vx,Hx,Wx (ev) | vsubsh Vx,Hx,Wx (F3),(ev) +5d: vminph Vx,Hx,Wx (ev) | vminsh Vx,Hx,Wx (F3),(ev) +5e: vdivph Vx,Hx,Wx (ev) | vdivsh Vx,Hx,Wx (F3),(ev) +5f: vmaxph Vx,Hx,Wx (ev) | vmaxsh Vx,Hx,Wx (F3),(ev) +6e: vmovw Vx,Wx (66),(ev) +78: vcvttph2udq Vx,Wx (ev) | vcvttph2uqq Vx,Wx (66),(ev) | vcvttsh2usi Vx,Wx (F3),(ev) +79: vcvtph2udq Vx,Wx (ev) | vcvtph2uqq Vx,Wx (66),(ev) | vcvtsh2usi Vx,Wx (F3),(ev) +7a: vcvttph2qq Vx,Wx (66),(ev) | vcvtudq2ph Vx,Wx (F2),(ev) | vcvtuqq2ph Vx,Wx (F2),(ev) +7b: vcvtph2qq Vx,Wx (66),(ev) | vcvtusi2sh Vx,Hx,Wx (F3),(ev) +7c: vcvttph2uw Vx,Wx (ev) | vcvttph2w Vx,Wx (66),(ev) +7d: vcvtph2uw Vx,Wx (ev) | vcvtph2w Vx,Wx (66),(ev) | vcvtuw2ph Vx,Wx (F2),(ev) | vcvtw2ph Vx,Wx (F3),(ev) +7e: vmovw Wx,Vx (66),(ev) +EndTable + +Table: EVEX map 6 +Referrer: +AVXcode: 6 +13: vcvtph2psx Vx,Wx (66),(ev) | vcvtsh2ss Vx,Hx,Wx (ev) +2c: vscalefph Vx,Hx,Wx (66),(ev) +2d: vscalefsh Vx,Hx,Wx (66),(ev) +42: vgetexpph Vx,Wx (66),(ev) +43: vgetexpsh Vx,Hx,Wx (66),(ev) +4c: vrcpph Vx,Wx (66),(ev) +4d: vrcpsh Vx,Hx,Wx (66),(ev) +4e: vrsqrtph Vx,Wx (66),(ev) +4f: vrsqrtsh Vx,Hx,Wx (66),(ev) +56: vfcmaddcph Vx,Hx,Wx (F2),(ev) | vfmaddcph Vx,Hx,Wx (F3),(ev) +57: vfcmaddcsh Vx,Hx,Wx (F2),(ev) | vfmaddcsh Vx,Hx,Wx (F3),(ev) +96: vfmaddsub132ph Vx,Hx,Wx (66),(ev) +97: vfmsubadd132ph Vx,Hx,Wx (66),(ev) +98: vfmadd132ph Vx,Hx,Wx (66),(ev) +99: vfmadd132sh Vx,Hx,Wx (66),(ev) +9a: vfmsub132ph Vx,Hx,Wx (66),(ev) +9b: vfmsub132sh Vx,Hx,Wx (66),(ev) +9c: vfnmadd132ph Vx,Hx,Wx (66),(ev) +9d: vfnmadd132sh Vx,Hx,Wx (66),(ev) +9e: vfnmsub132ph Vx,Hx,Wx (66),(ev) +9f: vfnmsub132sh Vx,Hx,Wx (66),(ev) +a6: vfmaddsub213ph Vx,Hx,Wx (66),(ev) +a7: vfmsubadd213ph Vx,Hx,Wx (66),(ev) +a8: vfmadd213ph Vx,Hx,Wx (66),(ev) +a9: vfmadd213sh Vx,Hx,Wx (66),(ev) +aa: vfmsub213ph Vx,Hx,Wx (66),(ev) +ab: vfmsub213sh Vx,Hx,Wx (66),(ev) +ac: vfnmadd213ph Vx,Hx,Wx (66),(ev) +ad: vfnmadd213sh Vx,Hx,Wx (66),(ev) +ae: vfnmsub213ph Vx,Hx,Wx (66),(ev) +af: vfnmsub213sh Vx,Hx,Wx (66),(ev) +b6: vfmaddsub231ph Vx,Hx,Wx (66),(ev) +b7: vfmsubadd231ph Vx,Hx,Wx (66),(ev) +b8: vfmadd231ph Vx,Hx,Wx (66),(ev) +b9: vfmadd231sh Vx,Hx,Wx (66),(ev) +ba: vfmsub231ph Vx,Hx,Wx (66),(ev) +bb: vfmsub231sh Vx,Hx,Wx (66),(ev) +bc: vfnmadd231ph Vx,Hx,Wx (66),(ev) +bd: vfnmadd231sh Vx,Hx,Wx (66),(ev) +be: vfnmsub231ph Vx,Hx,Wx (66),(ev) +bf: vfnmsub231sh Vx,Hx,Wx (66),(ev) +d6: vfcmulcph Vx,Hx,Wx (F2),(ev) | vfmulcph Vx,Hx,Wx (F3),(ev) +d7: vfcmulcsh Vx,Hx,Wx (F2),(ev) | vfmulcsh Vx,Hx,Wx (F3),(ev) EndTable GrpTable: Grp1 @@ -970,7 +1055,7 @@ GrpTable: Grp7 2: LGDT Ms | XGETBV (000),(11B) | XSETBV (001),(11B) | VMFUNC (100),(11B) | XEND (101)(11B) | XTEST (110)(11B) | ENCLU (111),(11B) 3: LIDT Ms 4: SMSW Mw/Rv -5: rdpkru (110),(11B) | wrpkru (111),(11B) | SAVEPREVSSP (F3),(010),(11B) | RSTORSSP Mq (F3) | SETSSBSY (F3),(000),(11B) +5: rdpkru (110),(11B) | wrpkru (111),(11B) | SAVEPREVSSP (F3),(010),(11B) | RSTORSSP Mq (F3) | SETSSBSY (F3),(000),(11B) | CLUI (F3),(110),(11B) | SERIALIZE (000),(11B) | STUI (F3),(111),(11B) | TESTUI (F3)(101)(11B) | UIRET (F3),(100),(11B) | XRESLDTRK (F2),(000),(11B) | XSUSLDTRK (F2),(001),(11B) 6: LMSW Ew 7: INVLPG Mb | SWAPGS (o64),(000),(11B) | RDTSCP (001),(11B) EndTable @@ -987,7 +1072,7 @@ GrpTable: Grp9 3: xrstors 4: xsavec 5: xsaves -6: VMPTRLD Mq | VMCLEAR Mq (66) | VMXON Mq (F3) | RDRAND Rv (11B) +6: VMPTRLD Mq | VMCLEAR Mq (66) | VMXON Mq (F3) | RDRAND Rv (11B) | SENDUIPI Gq (F3) 7: VMPTRST Mq | VMPTRST Mq (F3) | RDSEED Rv (11B) EndTable diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 026031b3b782..17a492c27306 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -615,6 +615,7 @@ static bool memremap_is_efi_data(resource_size_t phys_addr, static bool memremap_is_setup_data(resource_size_t phys_addr, unsigned long size) { + struct setup_indirect *indirect; struct setup_data *data; u64 paddr, paddr_next; @@ -627,6 +628,10 @@ static bool memremap_is_setup_data(resource_size_t phys_addr, data = memremap(paddr, sizeof(*data), MEMREMAP_WB | MEMREMAP_DEC); + if (!data) { + pr_warn("failed to memremap setup_data entry\n"); + return false; + } paddr_next = data->next; len = data->len; @@ -636,10 +641,21 @@ static bool memremap_is_setup_data(resource_size_t phys_addr, return true; } - if (data->type == SETUP_INDIRECT && - ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { - paddr = ((struct setup_indirect *)data->data)->addr; - len = ((struct setup_indirect *)data->data)->len; + if (data->type == SETUP_INDIRECT) { + memunmap(data); + data = memremap(paddr, sizeof(*data) + len, + MEMREMAP_WB | MEMREMAP_DEC); + if (!data) { + pr_warn("failed to memremap indirect setup_data\n"); + return false; + } + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) { + paddr = indirect->addr; + len = indirect->len; + } } memunmap(data); @@ -660,22 +676,51 @@ static bool memremap_is_setup_data(resource_size_t phys_addr, static bool __init early_memremap_is_setup_data(resource_size_t phys_addr, unsigned long size) { + struct setup_indirect *indirect; struct setup_data *data; u64 paddr, paddr_next; paddr = boot_params.hdr.setup_data; while (paddr) { - unsigned int len; + unsigned int len, size; if (phys_addr == paddr) return true; data = early_memremap_decrypted(paddr, sizeof(*data)); + if (!data) { + pr_warn("failed to early memremap setup_data entry\n"); + return false; + } + + size = sizeof(*data); paddr_next = data->next; len = data->len; - early_memunmap(data, sizeof(*data)); + if ((phys_addr > paddr) && (phys_addr < (paddr + len))) { + early_memunmap(data, sizeof(*data)); + return true; + } + + if (data->type == SETUP_INDIRECT) { + size += len; + early_memunmap(data, sizeof(*data)); + data = early_memremap_decrypted(paddr, size); + if (!data) { + pr_warn("failed to early memremap indirect setup_data\n"); + return false; + } + + indirect = (struct setup_indirect *)data->data; + + if (indirect->type != SETUP_INDIRECT) { + paddr = indirect->addr; + len = indirect->len; + } + } + + early_memunmap(data, size); if ((phys_addr > paddr) && (phys_addr < (paddr + len))) return true; diff --git a/arch/x86/mm/maccess.c b/arch/x86/mm/maccess.c index 92ec176a7293..5a53c2cc169c 100644 --- a/arch/x86/mm/maccess.c +++ b/arch/x86/mm/maccess.c @@ -4,11 +4,6 @@ #include <linux/kernel.h> #ifdef CONFIG_X86_64 -static __always_inline u64 canonical_address(u64 vaddr, u8 vaddr_bits) -{ - return ((s64)vaddr << (64 - vaddr_bits)) >> (64 - vaddr_bits); -} - bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) { unsigned long vaddr = (unsigned long)unsafe_src; @@ -19,7 +14,7 @@ bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) * we also need to include the userspace guard page. */ return vaddr >= TASK_SIZE_MAX + PAGE_SIZE && - canonical_address(vaddr, boot_cpu_data.x86_virt_bits) == vaddr; + __is_canonical_address(vaddr, boot_cpu_data.x86_virt_bits); } #else bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index 2b2d018ea345..6169053c2854 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -177,25 +177,6 @@ void __init sme_map_bootdata(char *real_mode_data) __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true); } -void __init sme_early_init(void) -{ - unsigned int i; - - if (!sme_me_mask) - return; - - early_pmd_flags = __sme_set(early_pmd_flags); - - __supported_pte_mask = __sme_set(__supported_pte_mask); - - /* Update the protection map with memory encryption mask */ - for (i = 0; i < ARRAY_SIZE(protection_map); i++) - protection_map[i] = pgprot_encrypted(protection_map[i]); - - if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) - swiotlb_force = SWIOTLB_FORCE; -} - void __init sev_setup_arch(void) { phys_addr_t total_mem = memblock_phys_mem_size(); @@ -256,7 +237,17 @@ static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot) return pfn; } -void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc) +static bool amd_enc_tlb_flush_required(bool enc) +{ + return true; +} + +static bool amd_enc_cache_flush_required(void) +{ + return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT); +} + +static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc) { #ifdef CONFIG_PARAVIRT unsigned long sz = npages << PAGE_SHIFT; @@ -287,6 +278,19 @@ void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc) #endif } +static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc) +{ +} + +/* Return true unconditionally: return value doesn't matter for the SEV side */ +static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc) +{ + if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) + enc_dec_hypercall(vaddr, npages, enc); + + return true; +} + static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) { pgprot_t old_prot, new_prot; @@ -392,7 +396,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, ret = 0; - notify_range_enc_status_changed(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc); + early_set_mem_enc_dec_hypercall(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc); out: __flush_tlb_all(); return ret; @@ -410,7 +414,31 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc) { - notify_range_enc_status_changed(vaddr, npages, enc); + enc_dec_hypercall(vaddr, npages, enc); +} + +void __init sme_early_init(void) +{ + unsigned int i; + + if (!sme_me_mask) + return; + + early_pmd_flags = __sme_set(early_pmd_flags); + + __supported_pte_mask = __sme_set(__supported_pte_mask); + + /* Update the protection map with memory encryption mask */ + for (i = 0; i < ARRAY_SIZE(protection_map); i++) + protection_map[i] = pgprot_encrypted(protection_map[i]); + + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + swiotlb_force = SWIOTLB_FORCE; + + x86_platform.guest.enc_status_change_prepare = amd_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish; + x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required; + x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required; } void __init mem_encrypt_free_decrypted_mem(void) diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 3f0abb403340..b43bc24d2bb6 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -44,6 +44,7 @@ #include <asm/setup.h> #include <asm/sections.h> #include <asm/cmdline.h> +#include <asm/coco.h> #include "mm_internal.h" @@ -565,8 +566,7 @@ void __init sme_enable(struct boot_params *bp) } else { /* SEV state cannot be controlled by a command line option */ sme_me_mask = me_mask; - physical_mask &= ~sme_me_mask; - return; + goto out; } /* @@ -600,6 +600,10 @@ void __init sme_enable(struct boot_params *bp) sme_me_mask = 0; else sme_me_mask = active_by_default ? me_mask : 0; - - physical_mask &= ~sme_me_mask; +out: + if (sme_me_mask) { + physical_mask &= ~sme_me_mask; + cc_set_vendor(CC_VENDOR_AMD); + cc_set_mask(sme_me_mask); + } } diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index c6b1213086d6..e8b061557887 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -738,17 +738,6 @@ void __init x86_numa_init(void) numa_init(dummy_numa_init); } -static void __init init_memory_less_node(int nid) -{ - /* Allocate and initialize node data. Memory-less node is now online.*/ - alloc_node_data(nid); - free_area_init_memoryless_node(nid); - - /* - * All zonelists will be built later in start_kernel() after per cpu - * areas are initialized. - */ -} /* * A node may exist which has one or more Generic Initiators but no CPUs and no @@ -766,9 +755,18 @@ void __init init_gi_nodes(void) { int nid; + /* + * Exclude this node from + * bringup_nonboot_cpus + * cpu_up + * __try_online_node + * register_one_node + * because node_subsys is not initialized yet. + * TODO remove dependency on node_online + */ for_each_node_state(nid, N_GENERIC_INITIATOR) if (!node_online(nid)) - init_memory_less_node(nid); + node_set_online(nid); } /* @@ -798,8 +796,17 @@ void __init init_cpu_to_node(void) if (node == NUMA_NO_NODE) continue; + /* + * Exclude this node from + * bringup_nonboot_cpus + * cpu_up + * __try_online_node + * register_one_node + * because node_subsys is not initialized yet. + * TODO remove dependency on node_online + */ if (!node_online(node)) - init_memory_less_node(node); + node_set_online(node); numa_set_node(cpu, node); } diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index b4072115c8ef..abf5ed76e4b7 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -1989,6 +1989,7 @@ int set_memory_global(unsigned long addr, int numpages) */ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) { + pgprot_t empty = __pgprot(0); struct cpa_data cpa; int ret; @@ -1999,18 +2000,20 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) memset(&cpa, 0, sizeof(cpa)); cpa.vaddr = &addr; cpa.numpages = numpages; - cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0); - cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC); + cpa.mask_set = enc ? pgprot_encrypted(empty) : pgprot_decrypted(empty); + cpa.mask_clr = enc ? pgprot_decrypted(empty) : pgprot_encrypted(empty); cpa.pgd = init_mm.pgd; /* Must avoid aliasing mappings in the highmem code */ kmap_flush_unused(); vm_unmap_aliases(); - /* - * Before changing the encryption attribute, we need to flush caches. - */ - cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT)); + /* Flush the caches as needed before changing the encryption attribute. */ + if (x86_platform.guest.enc_tlb_flush_required(enc)) + cpa_flush(&cpa, x86_platform.guest.enc_cache_flush_required()); + + /* Notify hypervisor that we are about to set/clr encryption attribute. */ + x86_platform.guest.enc_status_change_prepare(addr, numpages, enc); ret = __change_page_attr_set_clr(&cpa, 1); @@ -2023,11 +2026,11 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) */ cpa_flush(&cpa, 0); - /* - * Notify hypervisor that a given memory range is mapped encrypted - * or decrypted. - */ - notify_range_enc_status_changed(addr, numpages, enc); + /* Notify hypervisor that we have successfully set/clr encryption attribute. */ + if (!ret) { + if (!x86_platform.guest.enc_status_change_finish(addr, numpages, enc)) + ret = -EIO; + } return ret; } @@ -2121,12 +2124,6 @@ int set_pages_array_wc(struct page **pages, int numpages) } EXPORT_SYMBOL(set_pages_array_wc); -int set_pages_array_wt(struct page **pages, int numpages) -{ - return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WT); -} -EXPORT_SYMBOL_GPL(set_pages_array_wt); - int set_pages_wb(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2b1e266ff95c..0ecb140864b2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -394,7 +394,7 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) u8 *prog = *pprog; #ifdef CONFIG_RETPOLINE - if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_AMD)) { + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { EMIT_LFENCE(); EMIT2(0xFF, 0xE0 + reg); } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig index 40d6a06e41c8..ead7e5b3a975 100644 --- a/arch/x86/um/Kconfig +++ b/arch/x86/um/Kconfig @@ -8,6 +8,7 @@ endmenu config UML_X86 def_bool y + select ARCH_BINFMT_ELF_EXTRA_PHDRS if X86_32 config 64BIT bool "64-bit kernel" if "$(SUBARCH)" = "x86" diff --git a/arch/x86/um/user-offsets.c b/arch/x86/um/user-offsets.c index bae61554abcc..e54a9814ccf1 100644 --- a/arch/x86/um/user-offsets.c +++ b/arch/x86/um/user-offsets.c @@ -8,12 +8,11 @@ #define __FRAME_OFFSETS #include <linux/ptrace.h> #include <asm/types.h> +#include <linux/kbuild.h> -#define DEFINE(sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val)) - -#define DEFINE_LONGS(sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val/sizeof(unsigned long))) +#define DEFINE_LONGS(sym, val) \ + COMMENT(#val " / sizeof(unsigned long)"); \ + DEFINE(sym, val / sizeof(unsigned long)) void foo(void) { diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 6448c5071117..517a9d8d8f94 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -185,8 +185,7 @@ static int xen_cpu_dead_hvm(unsigned int cpu) if (xen_have_vector_callback && xen_feature(XENFEAT_hvm_safe_pvclock)) xen_teardown_timer(cpu); - - return 0; + return 0; } static bool no_vector_callback __initdata; @@ -248,6 +247,11 @@ static __init bool xen_x2apic_available(void) return x2apic_supported(); } +static bool __init msi_ext_dest_id(void) +{ + return cpuid_eax(xen_cpuid_base() + 4) & XEN_HVM_CPUID_EXT_DEST_ID; +} + static __init void xen_hvm_guest_late_init(void) { #ifdef CONFIG_XEN_PVH @@ -310,6 +314,7 @@ struct hypervisor_x86 x86_hyper_xen_hvm __initdata = { .init.x2apic_available = xen_x2apic_available, .init.init_mem_mapping = xen_hvm_init_mem_mapping, .init.guest_late_init = xen_hvm_guest_late_init, + .init.msi_ext_dest_id = msi_ext_dest_id, .runtime.pin_vcpu = xen_pin_vcpu, .ignore_nopv = true, }; diff --git a/arch/x86/xen/vga.c b/arch/x86/xen/vga.c index 31b1e3477cb6..14ea32e734d5 100644 --- a/arch/x86/xen/vga.c +++ b/arch/x86/xen/vga.c @@ -57,6 +57,14 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size) screen_info->rsvd_size = info->u.vesa_lfb.rsvd_size; screen_info->rsvd_pos = info->u.vesa_lfb.rsvd_pos; + if (size >= offsetof(struct dom0_vga_console_info, + u.vesa_lfb.ext_lfb_base) + + sizeof(info->u.vesa_lfb.ext_lfb_base) + && info->u.vesa_lfb.ext_lfb_base) { + screen_info->ext_lfb_base = info->u.vesa_lfb.ext_lfb_base; + screen_info->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; + } + if (info->video_type == XEN_VGATYPE_EFI_LFB) { screen_info->orig_video_isVGA = VIDEO_TYPE_EFI; break; @@ -66,14 +74,6 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size) u.vesa_lfb.mode_attrs) + sizeof(info->u.vesa_lfb.mode_attrs)) screen_info->vesa_attributes = info->u.vesa_lfb.mode_attrs; - - if (size >= offsetof(struct dom0_vga_console_info, - u.vesa_lfb.ext_lfb_base) - + sizeof(info->u.vesa_lfb.ext_lfb_base) - && info->u.vesa_lfb.ext_lfb_base) { - screen_info->ext_lfb_base = info->u.vesa_lfb.ext_lfb_base; - screen_info->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; - } break; } } diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 8ac599aa6d99..95bdbaaa5ae6 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -3,6 +3,7 @@ config XTENSA def_bool y select ARCH_32BIT_OFF_T select ARCH_HAS_BINFMT_FLAT if !MMU + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DMA_PREP_COHERENT if MMU select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU @@ -40,7 +41,6 @@ config XTENSA select IRQ_DOMAIN select MODULES_USE_ELF_RELA select PERF_USE_VMALLOC - select SET_FS select TRACE_IRQFLAGS_SUPPORT select VIRT_TO_BUS help diff --git a/arch/xtensa/include/asm/asm-uaccess.h b/arch/xtensa/include/asm/asm-uaccess.h index 7f6cf4151843..7cec869136e3 100644 --- a/arch/xtensa/include/asm/asm-uaccess.h +++ b/arch/xtensa/include/asm/asm-uaccess.h @@ -24,76 +24,6 @@ #include <asm/processor.h> /* - * These assembly macros mirror the C macros in asm/uaccess.h. They - * should always have identical functionality. See - * arch/xtensa/kernel/sys.S for usage. - */ - -#define KERNEL_DS 0 -#define USER_DS 1 - -/* - * get_fs reads current->thread.current_ds into a register. - * On Entry: - * <ad> anything - * <sp> stack - * On Exit: - * <ad> contains current->thread.current_ds - */ - .macro get_fs ad, sp - GET_CURRENT(\ad,\sp) -#if THREAD_CURRENT_DS > 1020 - addi \ad, \ad, TASK_THREAD - l32i \ad, \ad, THREAD_CURRENT_DS - TASK_THREAD -#else - l32i \ad, \ad, THREAD_CURRENT_DS -#endif - .endm - -/* - * set_fs sets current->thread.current_ds to some value. - * On Entry: - * <at> anything (temp register) - * <av> value to write - * <sp> stack - * On Exit: - * <at> destroyed (actually, current) - * <av> preserved, value to write - */ - .macro set_fs at, av, sp - GET_CURRENT(\at,\sp) - s32i \av, \at, THREAD_CURRENT_DS - .endm - -/* - * kernel_ok determines whether we should bypass addr/size checking. - * See the equivalent C-macro version below for clarity. - * On success, kernel_ok branches to a label indicated by parameter - * <success>. This implies that the macro falls through to the next - * insruction on an error. - * - * Note that while this macro can be used independently, we designed - * in for optimal use in the access_ok macro below (i.e., we fall - * through on error). - * - * On Entry: - * <at> anything (temp register) - * <success> label to branch to on success; implies - * fall-through macro on error - * <sp> stack pointer - * On Exit: - * <at> destroyed (actually, current->thread.current_ds) - */ - -#if ((KERNEL_DS != 0) || (USER_DS == 0)) -# error Assembly macro kernel_ok fails -#endif - .macro kernel_ok at, sp, success - get_fs \at, \sp - beqz \at, \success - .endm - -/* * user_ok determines whether the access to user-space memory is allowed. * See the equivalent C-macro version below for clarity. * @@ -147,7 +77,6 @@ * <at> destroyed */ .macro access_ok aa, as, at, sp, error - kernel_ok \at, \sp, .Laccess_ok_\@ user_ok \aa, \as, \at, \error .Laccess_ok_\@: .endm diff --git a/arch/xtensa/include/asm/current.h b/arch/xtensa/include/asm/current.h index 5d98a7ad4251..08010dbf5e09 100644 --- a/arch/xtensa/include/asm/current.h +++ b/arch/xtensa/include/asm/current.h @@ -26,6 +26,8 @@ static inline struct task_struct *get_current(void) #define current get_current() +register unsigned long current_stack_pointer __asm__("a1"); + #else #define GET_CURRENT(reg,sp) \ diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h index bd5aeb795567..8da562f5da73 100644 --- a/arch/xtensa/include/asm/pgtable.h +++ b/arch/xtensa/include/asm/pgtable.h @@ -241,6 +241,7 @@ static inline void paging_init(void) { } * The pmd contains the kernel virtual address of the pte page. */ #define pmd_page_vaddr(pmd) ((unsigned long)(pmd_val(pmd) & PAGE_MASK)) +#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) #define pmd_page(pmd) virt_to_page(pmd_val(pmd)) /* diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h index 37d3e9887fe7..abad7c3df46f 100644 --- a/arch/xtensa/include/asm/processor.h +++ b/arch/xtensa/include/asm/processor.h @@ -152,18 +152,12 @@ */ #define SPILL_SLOT_CALL12(sp, reg) (*(((unsigned long *)(sp)) - 16 + (reg))) -typedef struct { - unsigned long seg; -} mm_segment_t; - struct thread_struct { /* kernel's return address and stack pointer for context switching */ unsigned long ra; /* kernel's a0: return address and window call size */ unsigned long sp; /* kernel's a1: stack pointer */ - mm_segment_t current_ds; /* see uaccess.h for example uses */ - /* struct xtensa_cpuinfo info; */ unsigned long bad_vaddr; /* last user fault */ @@ -186,7 +180,6 @@ struct thread_struct { { \ ra: 0, \ sp: sizeof(init_stack) + (long) &init_stack, \ - current_ds: {0}, \ /*info: {0}, */ \ bad_vaddr: 0, \ bad_uaddr: 0, \ diff --git a/arch/xtensa/include/asm/stacktrace.h b/arch/xtensa/include/asm/stacktrace.h index fe06e8ed162b..a85e785a6288 100644 --- a/arch/xtensa/include/asm/stacktrace.h +++ b/arch/xtensa/include/asm/stacktrace.h @@ -19,14 +19,14 @@ struct stackframe { static __always_inline unsigned long *stack_pointer(struct task_struct *task) { - unsigned long *sp; + unsigned long sp; if (!task || task == current) - __asm__ __volatile__ ("mov %0, a1\n" : "=a"(sp)); + sp = current_stack_pointer; else - sp = (unsigned long *)task->thread.sp; + sp = task->thread.sp; - return sp; + return (unsigned long *)sp; } void walk_stackframe(unsigned long *sp, diff --git a/arch/xtensa/include/asm/thread_info.h b/arch/xtensa/include/asm/thread_info.h index a312333a9add..f6fcbba1d02f 100644 --- a/arch/xtensa/include/asm/thread_info.h +++ b/arch/xtensa/include/asm/thread_info.h @@ -52,8 +52,6 @@ struct thread_info { __u32 cpu; /* current CPU */ __s32 preempt_count; /* 0 => preemptable,< 0 => BUG*/ - mm_segment_t addr_limit; /* thread address space */ - unsigned long cpenable; #if XCHAL_HAVE_EXCLUSIVE /* result of the most recent exclusive store */ @@ -81,7 +79,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } /* how to get the thread information struct from C */ diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 75bd8fbf52ba..56aec6d504fe 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -19,31 +19,7 @@ #include <linux/prefetch.h> #include <asm/types.h> #include <asm/extable.h> - -/* - * The fs value determines whether argument validity checking should - * be performed or not. If get_fs() == USER_DS, checking is - * performed, with get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons (Data Segment Register?), these macros are - * grossly misnamed. - */ - -#define KERNEL_DS ((mm_segment_t) { 0 }) -#define USER_DS ((mm_segment_t) { 1 }) - -#define get_fs() (current->thread.current_ds) -#define set_fs(val) (current->thread.current_ds = (val)) - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) - -#define __kernel_ok (uaccess_kernel()) -#define __user_ok(addr, size) \ - (((size) <= TASK_SIZE)&&((addr) <= TASK_SIZE-(size))) -#define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) -#define access_ok(addr, size) __access_ok((unsigned long)(addr), (size)) - -#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE) +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They diff --git a/arch/xtensa/include/uapi/asm/shmbuf.h b/arch/xtensa/include/uapi/asm/shmbuf.h index 554a57a6a90f..bb8bdddae9b5 100644 --- a/arch/xtensa/include/uapi/asm/shmbuf.h +++ b/arch/xtensa/include/uapi/asm/shmbuf.h @@ -20,9 +20,12 @@ #ifndef _XTENSA_SHMBUF_H #define _XTENSA_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + struct shmid64_ds { struct ipc64_perm shm_perm; /* operation perms */ - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ unsigned long shm_atime; /* last attach time */ unsigned long shm_atime_high; unsigned long shm_dtime; /* last detach time */ diff --git a/arch/xtensa/include/uapi/asm/signal.h b/arch/xtensa/include/uapi/asm/signal.h index 79ddabaa4e5d..b8c824dd4b74 100644 --- a/arch/xtensa/include/uapi/asm/signal.h +++ b/arch/xtensa/include/uapi/asm/signal.h @@ -103,7 +103,7 @@ struct sigaction { typedef struct sigaltstack { void *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; #endif /* __ASSEMBLY__ */ diff --git a/arch/xtensa/kernel/asm-offsets.c b/arch/xtensa/kernel/asm-offsets.c index dc5c83cad9be..f1fd1390d069 100644 --- a/arch/xtensa/kernel/asm-offsets.c +++ b/arch/xtensa/kernel/asm-offsets.c @@ -87,7 +87,6 @@ int main(void) OFFSET(TI_STSTUS, thread_info, status); OFFSET(TI_CPU, thread_info, cpu); OFFSET(TI_PRE_COUNT, thread_info, preempt_count); - OFFSET(TI_ADDR_LIMIT, thread_info, addr_limit); /* struct thread_info (offset from start_struct) */ DEFINE(THREAD_RA, offsetof (struct task_struct, thread.ra)); @@ -108,8 +107,6 @@ int main(void) #endif DEFINE(THREAD_XTREGS_USER, offsetof (struct thread_info, xtregs_user)); DEFINE(XTREGS_USER_SIZE, sizeof(xtregs_user_t)); - DEFINE(THREAD_CURRENT_DS, offsetof (struct task_struct, \ - thread.current_ds)); /* struct mm_struct */ DEFINE(MM_USERS, offsetof(struct mm_struct, mm_users)); diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c index 15051a8a1539..529fe9245821 100644 --- a/arch/xtensa/kernel/irq.c +++ b/arch/xtensa/kernel/irq.c @@ -36,9 +36,8 @@ asmlinkage void do_IRQ(int hwirq, struct pt_regs *regs) #ifdef CONFIG_DEBUG_STACKOVERFLOW /* Debugging check for stack overflow: is there less than 1KB free? */ { - unsigned long sp; + unsigned long sp = current_stack_pointer; - __asm__ __volatile__ ("mov %0, a1\n" : "=a" (sp)); sp &= THREAD_SIZE - 1; if (unlikely(sp < (sizeof(thread_info) + 1024))) diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 8eb6ad1a3a1d..0f0e0724397f 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -108,13 +108,13 @@ static void simdisk_submit_bio(struct bio *bio) sector_t sector = bio->bi_iter.bi_sector; bio_for_each_segment(bvec, bio, iter) { - char *buffer = kmap_atomic(bvec.bv_page) + bvec.bv_offset; + char *buffer = bvec_kmap_local(&bvec); unsigned len = bvec.bv_len >> SECTOR_SHIFT; simdisk_transfer(dev, sector, len, buffer, bio_data_dir(bio) == WRITE); sector += len; - kunmap_atomic(buffer); + kunmap_local(buffer); } bio_endio(bio); |