Merge tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Borislav Petkov:
 "Extraordinary embargoed times call for extraordinary measures. That's
  why this week's x86/urgent branch is larger than usual, containing all
  the known fallout fixes after the SRSO mitigation got merged.

  I know, it is a bit late in the game but everyone who has reported a
  bug stemming from the SRSO pile, has tested that branch and has
  confirmed that it fixes their bug.

  Also, I've run it on every possible hardware I have and it is looking
  good. It is running on this very machine while I'm typing, for 2 days
  now without an issue. Famous last words...

   - Use LEA ...%rsp instead of ADD %rsp in the Zen1/2 SRSO return
     sequence as latter clobbers flags which interferes with fastop
     emulation in KVM, leading to guests freezing during boot

   - A fix for the DIV(0) quotient data leak on Zen1 to clear the
     divider buffers at the right time

   - Disable the SRSO mitigation on unaffected configurations as it got
     enabled there unnecessarily

   - Change .text section name to fix CONFIG_LTO_CLANG builds

   - Improve the optprobe indirect jmp check so that certain
     configurations can still be able to use optprobes at all

   - A serious and good scrubbing of the untraining routines by PeterZ:
      - Add proper speculation stopping traps so that objtool is happy
      - Adjust objtool to handle the new thunks
      - Make the thunk pointer assignable to the different untraining
        sequences at runtime, thus avoiding the alternative at the
        return thunk. It simplifies the code a bit too.
      - Add a entry_untrain_ret() main entry point which selects the
        respective untraining sequence
      - Rename things so that they're more clear
      - Fix stack validation with FRAME_POINTER=y builds

   - Fix static call patching to handle when a JMP to the return thunk
     is the last insn on the very last module memory page

   - Add more documentation about what each untraining routine does and
     why"

* tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/srso: Correct the mitigation status when SMT is disabled
  x86/static_call: Fix __static_call_fixup()
  objtool/x86: Fixup frame-pointer vs rethunk
  x86/srso: Explain the untraining sequences a bit more
  x86/cpu/kvm: Provide UNTRAIN_RET_VM
  x86/cpu: Cleanup the untrain mess
  x86/cpu: Rename srso_(.*)_alias to srso_alias_\1
  x86/cpu: Rename original retbleed methods
  x86/cpu: Clean up SRSO return thunk mess
  x86/alternative: Make custom return thunk unconditional
  objtool/x86: Fix SRSO mess
  x86/cpu: Fix up srso_safe_ret() and __x86_return_thunk()
  x86/cpu: Fix __x86_return_thunk symbol type
  x86/retpoline,kprobes: Skip optprobe check for indirect jumps with retpolines and IBT
  x86/retpoline,kprobes: Fix position of thunk sections with CONFIG_LTO_CLANG
  x86/srso: Disable the mitigation on unaffected configurations
  x86/CPU/AMD: Fix the DIV(0) initial fix attempt
  x86/retpoline: Don't clobber RFLAGS during srso_safe_ret()

2 files changed, 24 insertions, 4 deletions

diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 70f9d56f9305..7eca6a8abbb1 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -1329,3 +1329,4 @@ void noinstr amd_clear_divider(void)
 	asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)
 		     :: "a" (0), "d" (0), "r" (1));
 }
+EXPORT_SYMBOL_GPL(amd_clear_divider);
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index d02f73c5339d..f081d26616ac 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -63,6 +63,8 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd);
 
 static DEFINE_MUTEX(spec_ctrl_mutex);
 
+void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
+
 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
 static void update_spec_ctrl(u64 val)
 {
@@ -165,6 +167,11 @@ void __init cpu_select_mitigations(void)
 	md_clear_select_mitigation();
 	srbds_select_mitigation();
 	l1d_flush_select_mitigation();
+
+	/*
+	 * srso_select_mitigation() depends and must run after
+	 * retbleed_select_mitigation().
+	 */
 	srso_select_mitigation();
 	gds_select_mitigation();
 }
@@ -1035,6 +1042,9 @@ do_cmd_auto:
 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
 		setup_force_cpu_cap(X86_FEATURE_UNRET);
 
+		if (IS_ENABLED(CONFIG_RETHUNK))
+			x86_return_thunk = retbleed_return_thunk;
+
 		if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 		    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 			pr_err(RETBLEED_UNTRAIN_MSG);
@@ -1044,6 +1054,7 @@ do_cmd_auto:
 
 	case RETBLEED_MITIGATION_IBPB:
 		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
+		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
 		mitigate_smt = true;
 		break;
 
@@ -2417,9 +2428,10 @@ static void __init srso_select_mitigation(void)
 		 * Zen1/2 with SMT off aren't vulnerable after the right
 		 * IBPB microcode has been applied.
 		 */
-		if ((boot_cpu_data.x86 < 0x19) &&
-		    (!cpu_smt_possible() || (cpu_smt_control == CPU_SMT_DISABLED)))
+		if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
 			setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
+			return;
+		}
 	}
 
 	if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
@@ -2448,11 +2460,15 @@ static void __init srso_select_mitigation(void)
 			 * like ftrace, static_call, etc.
 			 */
 			setup_force_cpu_cap(X86_FEATURE_RETHUNK);
+			setup_force_cpu_cap(X86_FEATURE_UNRET);
 
-			if (boot_cpu_data.x86 == 0x19)
+			if (boot_cpu_data.x86 == 0x19) {
 				setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
-			else
+				x86_return_thunk = srso_alias_return_thunk;
+			} else {
 				setup_force_cpu_cap(X86_FEATURE_SRSO);
+				x86_return_thunk = srso_return_thunk;
+			}
 			srso_mitigation = SRSO_MITIGATION_SAFE_RET;
 		} else {
 			pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
@@ -2696,6 +2712,9 @@ static ssize_t retbleed_show_state(char *buf)
 
 static ssize_t srso_show_state(char *buf)
 {
+	if (boot_cpu_has(X86_FEATURE_SRSO_NO))
+		return sysfs_emit(buf, "Mitigation: SMT disabled\n");
+
 	return sysfs_emit(buf, "%s%s\n",
 			  srso_strings[srso_mitigation],
 			  (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode"));