From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Mon, 26 Feb 2018 17:34:21 +0000 (-0800)
Subject: Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git... 
X-Git-Tag: v4.16-rc4~37
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=85a2d939c05965ab9e849735436a3c8d3538dc75;p=thirdparty%2Fkernel%2Fstable.git

Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Thomas Gleixner:
 "Yet another pile of melted spectrum related changes:

   - sanitize the array_index_nospec protection mechanism: Remove the
     overengineered array_index_nospec_mask_check() magic and allow
     const-qualified types as index to avoid temporary storage in a
     non-const local variable.

   - make the microcode loader more robust by properly propagating error
     codes. Provide information about new feature bits after micro code
     was updated so administrators can act upon.

   - optimizations of the entry ASM code which reduce code footprint and
     make the code simpler and faster.

   - fix the {pmd,pud}_{set,clear}_flags() implementations to work
     properly on paravirt kernels by removing the address translation
     operations.

   - revert the harmful vmexit_fill_RSB() optimization

   - use IBRS around firmware calls

   - teach objtool about retpolines and add annotations for indirect
     jumps and calls.

   - explicitly disable jumplabel patching in __init code and handle
     patching failures properly instead of silently ignoring them.

   - remove indirect paravirt calls for writing the speculation control
     MSR as these calls are obviously proving the same attack vector
     which is tried to be mitigated.

   - a few small fixes which address build issues with recent compiler
     and assembler versions"

* 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (38 commits)
  KVM/VMX: Optimize vmx_vcpu_run() and svm_vcpu_run() by marking the RDMSR path as unlikely()
  KVM/x86: Remove indirect MSR op calls from SPEC_CTRL
  objtool, retpolines: Integrate objtool with retpoline support more closely
  x86/entry/64: Simplify ENCODE_FRAME_POINTER
  extable: Make init_kernel_text() global
  jump_label: Warn on failed jump_label patching attempt
  jump_label: Explicitly disable jump labels in __init code
  x86/entry/64: Open-code switch_to_thread_stack()
  x86/entry/64: Move ASM_CLAC to interrupt_entry()
  x86/entry/64: Remove 'interrupt' macro
  x86/entry/64: Move the switch_to_thread_stack() call to interrupt_entry()
  x86/entry/64: Move ENTER_IRQ_STACK from interrupt macro to interrupt_entry
  x86/entry/64: Move PUSH_AND_CLEAR_REGS from interrupt macro to helper function
  x86/speculation: Move firmware_restrict_branch_speculation_*() from C to CPP
  objtool: Add module specific retpoline rules
  objtool: Add retpoline validation
  objtool: Use existing global variables for options
  x86/mm/sme, objtool: Annotate indirect call in sme_encrypt_execute()
  x86/boot, objtool: Annotate indirect jump in secondary_startup_64()
  x86/paravirt, objtool: Annotate indirect calls
  ...
---

85a2d939c05965ab9e849735436a3c8d3538dc75
diff --cc arch/x86/include/asm/nospec-branch.h
index 81a1be3265711,b7063cfa19f97..d0dabeae05059
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@@ -163,55 -238,41 +238,78 @@@ static inline void vmexit_fill_RSB(void
  #endif
  }
  
+ #define alternative_msr_write(_msr, _val, _feature)		\
+ 	asm volatile(ALTERNATIVE("",				\
+ 				 "movl %[msr], %%ecx\n\t"	\
+ 				 "movl %[val], %%eax\n\t"	\
+ 				 "movl $0, %%edx\n\t"		\
+ 				 "wrmsr",			\
+ 				 _feature)			\
+ 		     : : [msr] "i" (_msr), [val] "i" (_val)	\
+ 		     : "eax", "ecx", "edx", "memory")
+ 
  static inline void indirect_branch_prediction_barrier(void)
  {
- 	asm volatile(ALTERNATIVE("",
- 				 "movl %[msr], %%ecx\n\t"
- 				 "movl %[val], %%eax\n\t"
- 				 "movl $0, %%edx\n\t"
- 				 "wrmsr",
- 				 X86_FEATURE_USE_IBPB)
- 		     : : [msr] "i" (MSR_IA32_PRED_CMD),
- 			 [val] "i" (PRED_CMD_IBPB)
- 		     : "eax", "ecx", "edx", "memory");
+ 	alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
+ 			      X86_FEATURE_USE_IBPB);
  }
  
+ /*
+  * With retpoline, we must use IBRS to restrict branch prediction
+  * before calling into firmware.
+  *
+  * (Implemented as CPP macros due to header hell.)
+  */
+ #define firmware_restrict_branch_speculation_start()			\
+ do {									\
+ 	preempt_disable();						\
+ 	alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,	\
+ 			      X86_FEATURE_USE_IBRS_FW);			\
+ } while (0)
+ 
+ #define firmware_restrict_branch_speculation_end()			\
+ do {									\
+ 	alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,			\
+ 			      X86_FEATURE_USE_IBRS_FW);			\
+ 	preempt_enable();						\
+ } while (0)
+ 
  #endif /* __ASSEMBLY__ */
 +
 +/*
 + * Below is used in the eBPF JIT compiler and emits the byte sequence
 + * for the following assembly:
 + *
 + * With retpolines configured:
 + *
 + *    callq do_rop
 + *  spec_trap:
 + *    pause
 + *    lfence
 + *    jmp spec_trap
 + *  do_rop:
 + *    mov %rax,(%rsp)
 + *    retq
 + *
 + * Without retpolines configured:
 + *
 + *    jmp *%rax
 + */
 +#ifdef CONFIG_RETPOLINE
 +# define RETPOLINE_RAX_BPF_JIT_SIZE	17
 +# define RETPOLINE_RAX_BPF_JIT()				\
 +	EMIT1_off32(0xE8, 7);	 /* callq do_rop */		\
 +	/* spec_trap: */					\
 +	EMIT2(0xF3, 0x90);       /* pause */			\
 +	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
 +	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
 +	/* do_rop: */						\
 +	EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */	\
 +	EMIT1(0xC3);             /* retq */
 +#else
 +# define RETPOLINE_RAX_BPF_JIT_SIZE	2
 +# define RETPOLINE_RAX_BPF_JIT()				\
 +	EMIT2(0xFF, 0xE0);	 /* jmp *%rax */
 +#endif
 +
  #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */