From 4c382d723edce1b3c72b55b1b505cf5526a56afc Mon Sep 17 00:00:00 2001 From: Brian Gerst Date: Tue, 24 Jan 2023 13:40:19 -0500 Subject: x86/vdso: Move VDSO image init to vdso2c generated code Generate an init function for each VDSO image, replacing init_vdso() and sysenter_setup(). Signed-off-by: Brian Gerst Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20230124184019.26850-1-brgerst@gmail.com --- arch/x86/entry/vdso/vdso2c.h | 6 ++++++ arch/x86/entry/vdso/vdso32-setup.c | 9 --------- arch/x86/entry/vdso/vma.c | 19 ++++--------------- arch/x86/include/asm/processor.h | 1 - arch/x86/include/asm/vdso.h | 2 +- arch/x86/kernel/cpu/common.c | 1 - 6 files changed, 11 insertions(+), 27 deletions(-) diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 5264daa8859f..67b3e37576a6 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -179,6 +179,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "/* AUTOMATICALLY GENERATED -- DO NOT EDIT */\n\n"); fprintf(outfile, "#include \n"); + fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "\n"); @@ -218,5 +219,10 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", required_syms[i].name, (int64_t)syms[i]); } + fprintf(outfile, "};\n\n"); + fprintf(outfile, "static __init int init_%s(void) {\n", image_name); + fprintf(outfile, "\treturn init_vdso_image(&%s);\n", image_name); fprintf(outfile, "};\n"); + fprintf(outfile, "subsys_initcall(init_%s);\n", image_name); + } diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c index 43842fade8fa..3b300a773c7e 100644 --- a/arch/x86/entry/vdso/vdso32-setup.c +++ b/arch/x86/entry/vdso/vdso32-setup.c @@ -51,17 +51,8 @@ __setup("vdso32=", vdso32_setup); __setup_param("vdso=", vdso_setup, vdso32_setup, 0); #endif -int __init sysenter_setup(void) -{ - init_vdso_image(&vdso_image_32); - - return 0; -} - #ifdef CONFIG_X86_64 -subsys_initcall(sysenter_setup); - #ifdef CONFIG_SYSCTL /* Register vsyscall32 into the ABI table */ #include diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index b8f3f9b9e53c..2738eb28cb2e 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -44,13 +44,16 @@ unsigned int vclocks_used __read_mostly; unsigned int __read_mostly vdso64_enabled = 1; #endif -void __init init_vdso_image(const struct vdso_image *image) +int __init init_vdso_image(const struct vdso_image *image) { + BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); BUG_ON(image->size % PAGE_SIZE != 0); apply_alternatives((struct alt_instr *)(image->data + image->alt), (struct alt_instr *)(image->data + image->alt + image->alt_len)); + + return 0; } static const struct vm_special_mapping vvar_mapping; @@ -418,18 +421,4 @@ static __init int vdso_setup(char *s) return 1; } __setup("vdso=", vdso_setup); - -static int __init init_vdso(void) -{ - BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); - - init_vdso_image(&vdso_image_64); - -#ifdef CONFIG_X86_X32_ABI - init_vdso_image(&vdso_image_x32); -#endif - - return 0; -} -subsys_initcall(init_vdso); #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 4e35c66edeb7..2dd0c8a85e62 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -542,7 +542,6 @@ enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL}; extern void enable_sep_cpu(void); -extern int sysenter_setup(void); /* Defined in head.S */ diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index 2963a2f5dbc4..d7f6592b74a9 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -45,7 +45,7 @@ extern const struct vdso_image vdso_image_x32; extern const struct vdso_image vdso_image_32; #endif -extern void __init init_vdso_image(const struct vdso_image *image); +extern int __init init_vdso_image(const struct vdso_image *image); extern int map_vdso_once(const struct vdso_image *image, unsigned long addr); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9cfca3d7d0e2..3a9043ef3d6b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1953,7 +1953,6 @@ void __init identify_boot_cpu(void) if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) pr_info("CET detected: Indirect Branch Tracking enabled\n"); #ifdef CONFIG_X86_32 - sysenter_setup(); enable_sep_cpu(); #endif cpu_detect_tlb(&boot_cpu_data); -- cgit v1.2.3 From 717cce3bdcf34705417f641bf2fcdf9b038ec36c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:14 +0100 Subject: x86/cpu: Provide the full setup for getcpu() on x86-32 setup_getcpu() configures two things: - it writes the current CPU & node information into MSR_TSC_AUX - it writes the same information as a GDT entry. By using the "full" setup_getcpu() on i386 it is possible to read the CPU information in userland via RDTSCP() or via LSL from the GDT. Provide an GDT_ENTRY_CPUNODE for x86-32 and make the setup function unconditionally available. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Reviewed-by: Roland Mainz Link: https://lore.kernel.org/r/20221125094216.3663444-2-bigeasy@linutronix.de --- arch/x86/include/asm/segment.h | 8 ++++---- arch/x86/kernel/cpu/common.c | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index c390a672d560..794f69625780 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -96,7 +96,7 @@ * * 26 - ESPFIX small SS * 27 - per-cpu [ offset to per-cpu data area ] - * 28 - unused + * 28 - VDSO getcpu * 29 - unused * 30 - unused * 31 - TSS for double fault handler @@ -119,6 +119,7 @@ #define GDT_ENTRY_ESPFIX_SS 26 #define GDT_ENTRY_PERCPU 27 +#define GDT_ENTRY_CPUNODE 28 #define GDT_ENTRY_DOUBLEFAULT_TSS 31 @@ -159,6 +160,8 @@ # define __KERNEL_PERCPU 0 #endif +#define __CPUNODE_SEG (GDT_ENTRY_CPUNODE*8 + 3) + #else /* 64-bit: */ #include @@ -226,8 +229,6 @@ #define GDT_ENTRY_TLS_ENTRIES 3 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES* 8) -#ifdef CONFIG_X86_64 - /* Bit size and mask of CPU number stored in the per CPU data (and TSC_AUX) */ #define VDSO_CPUNODE_BITS 12 #define VDSO_CPUNODE_MASK 0xfff @@ -265,7 +266,6 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) } #endif /* !__ASSEMBLY__ */ -#endif /* CONFIG_X86_64 */ #ifdef __KERNEL__ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 3a9043ef3d6b..a26ce206f7b0 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2124,7 +2124,6 @@ static void wait_for_master_cpu(int cpu) #endif } -#ifdef CONFIG_X86_64 static inline void setup_getcpu(int cpu) { unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu)); @@ -2146,6 +2145,7 @@ static inline void setup_getcpu(int cpu) write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S); } +#ifdef CONFIG_X86_64 static inline void ucode_cpu_init(int cpu) { if (cpu) @@ -2165,8 +2165,6 @@ static inline void tss_setup_ist(struct tss_struct *tss) #else /* CONFIG_X86_64 */ -static inline void setup_getcpu(int cpu) { } - static inline void ucode_cpu_init(int cpu) { show_ucode_info_early(); -- cgit v1.2.3 From 92d33063c081a82d25dd08a9cce03947c8ed9164 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:15 +0100 Subject: x86/vdso: Provide getcpu for x86-32. Wire up __vdso_getcpu() for x86-32. The 64bit version is reused with trivial modifications. Contrary to vclock_gettime.c there is no requirement to fake any defines in the case of 32bit VDSO on a 64bit kernel because the GDT entry from which the CPU and node information is read is always the native one. Adopt vdso_getcpu.c by: - removing the unneeded time* header files which lead to compile errors for 32bit. - adding segment.h which provides vdso_read_cpunode() and the defines required by it. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20221125094216.3663444-3-bigeasy@linutronix.de --- arch/x86/entry/vdso/Makefile | 3 ++- arch/x86/entry/vdso/vdso32/vdso32.lds.S | 1 + arch/x86/entry/vdso/vdso32/vgetcpu.c | 2 ++ arch/x86/entry/vdso/vgetcpu.c | 3 +-- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 arch/x86/entry/vdso/vdso32/vgetcpu.c diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 838613ac15b8..1506a22a4fb6 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -29,7 +29,7 @@ VDSO32-$(CONFIG_IA32_EMULATION) := y # files to link into the vdso vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o -vobjs32-y += vdso32/vclock_gettime.o +vobjs32-y += vdso32/vclock_gettime.o vdso32/vgetcpu.o vobjs-$(CONFIG_X86_SGX) += vsgx.o # files to link into kernel @@ -104,6 +104,7 @@ $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO CFLAGS_REMOVE_vclock_gettime.o = -pg CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg CFLAGS_REMOVE_vgetcpu.o = -pg +CFLAGS_REMOVE_vdso32/vgetcpu.o = -pg CFLAGS_REMOVE_vsgx.o = -pg # diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S index c7720995ab1a..8a3be07006bb 100644 --- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S +++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S @@ -28,6 +28,7 @@ VERSION __vdso_time; __vdso_clock_getres; __vdso_clock_gettime64; + __vdso_getcpu; }; LINUX_2.5 { diff --git a/arch/x86/entry/vdso/vdso32/vgetcpu.c b/arch/x86/entry/vdso/vdso32/vgetcpu.c new file mode 100644 index 000000000000..b777f84ffae9 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/vgetcpu.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../vgetcpu.c" diff --git a/arch/x86/entry/vdso/vgetcpu.c b/arch/x86/entry/vdso/vgetcpu.c index b88a82bbc359..0a9007c24056 100644 --- a/arch/x86/entry/vdso/vgetcpu.c +++ b/arch/x86/entry/vdso/vgetcpu.c @@ -7,8 +7,7 @@ #include #include -#include -#include +#include notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) -- cgit v1.2.3 From 5646bbd6684acf5c9b9dedb863b7d2f6f5a330fb Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:16 +0100 Subject: selftests: Emit a warning if getcpu() is missing on 32bit The VDSO implementation for getcpu() has been wired up on 32bit so warn if missing. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Shuah Khan Link: https://lore.kernel.org/r/20221125094216.3663444-4-bigeasy@linutronix.de --- tools/testing/selftests/x86/test_vsyscall.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c index 5b45e6986aea..47cab972807c 100644 --- a/tools/testing/selftests/x86/test_vsyscall.c +++ b/tools/testing/selftests/x86/test_vsyscall.c @@ -92,11 +92,8 @@ static void init_vdso(void) printf("[WARN]\tfailed to find time in vDSO\n"); vdso_getcpu = (getcpu_t)dlsym(vdso, "__vdso_getcpu"); - if (!vdso_getcpu) { - /* getcpu() was never wired up in the 32-bit vDSO. */ - printf("[%s]\tfailed to find getcpu in vDSO\n", - sizeof(long) == 8 ? "WARN" : "NOTE"); - } + if (!vdso_getcpu) + printf("[WARN]\tfailed to find getcpu in vDSO\n"); } static int init_vsys(void) -- cgit v1.2.3 From 877cff5296faa6ef312f8002216bbf73bea006c7 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 7 Feb 2023 11:46:33 +0100 Subject: x86/vdso: Fake 32bit VDSO build on 64bit compile for vgetcpu The 64bit register constrains in __arch_hweight64() cannot be fulfilled in a 32-bit build. The function is only declared but not used within vclock_gettime.c and gcc does not care. LLVM complains and aborts. Reportedly because it validates extended asm even if latter would get compiled out, see https://lore.kernel.org/r/Y%2BJ%2BUQ1vAKr6RHuH@dev-arch.thelio-3990X i.e., a long standing design difference between gcc and LLVM. Move the "fake a 32 bit kernel configuration" bits from vclock_gettime.c into a common header file. Use this from vclock_gettime.c and vgetcpu.c. [ bp: Add background info from Nathan. ] Fixes: 92d33063c081a ("x86/vdso: Provide getcpu for x86-32.") Reported-by: kernel test robot Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/Y+IsCWQdXEr8d9Vy@linutronix.de --- arch/x86/entry/vdso/vdso32/fake_32bit_build.h | 25 +++++++++++++++++++++++++ arch/x86/entry/vdso/vdso32/vclock_gettime.c | 27 +-------------------------- arch/x86/entry/vdso/vdso32/vgetcpu.c | 1 + 3 files changed, 27 insertions(+), 26 deletions(-) create mode 100644 arch/x86/entry/vdso/vdso32/fake_32bit_build.h diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h new file mode 100644 index 000000000000..db1b15f686e3 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef CONFIG_X86_64 + +/* + * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel + * configuration + */ +#undef CONFIG_64BIT +#undef CONFIG_X86_64 +#undef CONFIG_COMPAT +#undef CONFIG_PGTABLE_LEVELS +#undef CONFIG_ILLEGAL_POINTER_VALUE +#undef CONFIG_SPARSEMEM_VMEMMAP +#undef CONFIG_NR_CPUS +#undef CONFIG_PARAVIRT_XXL + +#define CONFIG_X86_32 1 +#define CONFIG_PGTABLE_LEVELS 2 +#define CONFIG_PAGE_OFFSET 0 +#define CONFIG_ILLEGAL_POINTER_VALUE 0 +#define CONFIG_NR_CPUS 1 + +#define BUILD_VDSO32_64 + +#endif diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c index 283ed9d00426..86981decfea8 100644 --- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c +++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c @@ -1,29 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 #define BUILD_VDSO32 - -#ifdef CONFIG_X86_64 - -/* - * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel - * configuration - */ -#undef CONFIG_64BIT -#undef CONFIG_X86_64 -#undef CONFIG_COMPAT -#undef CONFIG_PGTABLE_LEVELS -#undef CONFIG_ILLEGAL_POINTER_VALUE -#undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_NR_CPUS -#undef CONFIG_PARAVIRT_XXL - -#define CONFIG_X86_32 1 -#define CONFIG_PGTABLE_LEVELS 2 -#define CONFIG_PAGE_OFFSET 0 -#define CONFIG_ILLEGAL_POINTER_VALUE 0 -#define CONFIG_NR_CPUS 1 - -#define BUILD_VDSO32_64 - -#endif - +#include "fake_32bit_build.h" #include "../vclock_gettime.c" diff --git a/arch/x86/entry/vdso/vdso32/vgetcpu.c b/arch/x86/entry/vdso/vdso32/vgetcpu.c index b777f84ffae9..3a9791f5e998 100644 --- a/arch/x86/entry/vdso/vdso32/vgetcpu.c +++ b/arch/x86/entry/vdso/vdso32/vgetcpu.c @@ -1,2 +1,3 @@ // SPDX-License-Identifier: GPL-2.0 +#include "fake_32bit_build.h" #include "../vgetcpu.c" -- cgit v1.2.3 From dac0da428f8dfb23666ab4ea54a7768c152b857a Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 7 Feb 2023 16:13:24 +0100 Subject: x86/vdso: Fix -Wmissing-prototypes warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix those: In file included from arch/x86/entry/vdso/vdso32/vclock_gettime.c:4: arch/x86/entry/vdso/vdso32/../vclock_gettime.c:70:5: warning: no previous prototype for ‘__vdso_clock_gettime64’ [-Wmissing-prototypes] 70 | int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) | In file included from arch/x86/entry/vdso/vdso32/vgetcpu.c:3: arch/x86/entry/vdso/vdso32/../vgetcpu.c:13:1: warning: no previous prototype for ‘__vdso_getcpu’ [-Wmissing-prototypes] 13 | __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) | ^~~~~~~~~~~~~ Reported-by: kernel test robot Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/202302070742.iYcnoJwk-lkp@intel.com --- arch/x86/include/asm/vdso/gettimeofday.h | 2 ++ arch/x86/include/asm/vdso/processor.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 1936f21ed8cd..4cf6794f9d68 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -318,6 +318,8 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #define vdso_calc_delta vdso_calc_delta +int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts); + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index 57b1a7034c64..2cbce97d29ea 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -18,6 +18,10 @@ static __always_inline void cpu_relax(void) rep_nop(); } +struct getcpu_cache; + +notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); + #endif /* __ASSEMBLY__ */ #endif /* __ASM_VDSO_PROCESSOR_H */ -- cgit v1.2.3