x86/its: Add support for ITS-safe indirect thunk

commit 8754e67ad4ac692c67ff1f99c0d07156f04ae40c upstream.

Due to ITS, indirect branches in the lower half of a cacheline may be
vulnerable to branch target injection attack.

Introduce ITS-safe thunks to patch indirect branches in the lower half of
cacheline with the thunk. Also thunk any eBPF generated indirect branches
in emit_indirect_jump().

Below category of indirect branches are not mitigated:

- Indirect branches in the .init section are not mitigated because they are
  discarded after boot.
- Indirect branches that are explicitly marked retpoline-safe.

Note that retpoline also mitigates the indirect branches against ITS. This
is because the retpoline sequence fills an RSB entry before RET, and it
does not suffer from RSB-underflow part of the ITS.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d874ea22512b5cc08df79dda453d28cd8907cf91..4372657ab0d6fa7451f509ee0dffa10f6bfc8f34 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2610,6 +2610,17 @@ config MITIGATION_SPECTRE_BHI
          indirect branches.
          See <file:Documentation/admin-guide/hw-vuln/spectre.rst>
 
+config MITIGATION_ITS
+       bool "Enable Indirect Target Selection mitigation"
+       depends on CPU_SUP_INTEL && X86_64
+       depends on RETPOLINE && RETHUNK
+       default y
+       help
+         Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in
+         BPU on some Intel CPUs that may allow Spectre V2 style attacks. If
+         disabled, mitigation cannot be enabled via cmdline.
+         See <file:Documentation/admin-guide/hw-vuln/indirect-target-selection.rst>
+
 endif
 
 config ARCH_HAS_ADD_PAGES

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 0fde2feb16c62f65dabb3d1c6945aedaf096c204..8cc47fc986e9ce9ed2183fc83d7f335346cb36c0 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -468,6 +468,7 @@
 #define X86_FEATURE_BHI_CTRL           (21*32+ 2) /* "" BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW       (21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 5) /* "" Use thunk for indirect branches in lower half of cacheline */
 
 /*
  * BUG word(s)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 7e4c91eb8c10b5b69204c96e619ec985c8012d2b..8633f7bff764a73bb23fd6ba0e42b3961753c370 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -364,10 +364,14 @@
        ".long 999b\n\t"                                        \
        ".popsection\n\t"
 
+#define ITS_THUNK_SIZE 64
+
 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
+typedef u8 its_thunk_t[ITS_THUNK_SIZE];
 extern retpoline_thunk_t __x86_indirect_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
+extern its_thunk_t      __x86_indirect_its_thunk_array[];
 
 #ifdef CONFIG_RETHUNK
 extern void __x86_return_thunk(void);

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index aae7456ece070002c5f73bd68866fc2ef083a35e..9be01147727e9edf7e0e545a808450756a2a7ee9 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -521,7 +521,8 @@ static int emit_indirect(int op, int reg, u8 *bytes)
        return i;
 }
 
-static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
+                            void *call_dest, void *jmp_dest)
 {
        u8 op = insn->opcode.bytes[0];
        int i = 0;
@@ -542,7 +543,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
        switch (op) {
        case CALL_INSN_OPCODE:
                __text_gen_insn(bytes+i, op, addr+i,
-                               __x86_indirect_call_thunk_array[reg],
+                               call_dest,
                                CALL_INSN_SIZE);
                i += CALL_INSN_SIZE;
                break;
@@ -550,7 +551,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
        case JMP32_INSN_OPCODE:
 clang_jcc:
                __text_gen_insn(bytes+i, op, addr+i,
-                               __x86_indirect_jump_thunk_array[reg],
+                               jmp_dest,
                                JMP32_INSN_SIZE);
                i += JMP32_INSN_SIZE;
                break;
@@ -565,6 +566,35 @@ clang_jcc:
        return i;
 }
 
+static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+       return __emit_trampoline(addr, insn, bytes,
+                                __x86_indirect_call_thunk_array[reg],
+                                __x86_indirect_jump_thunk_array[reg]);
+}
+
+#ifdef CONFIG_MITIGATION_ITS
+static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+       return __emit_trampoline(addr, insn, bytes,
+                                __x86_indirect_its_thunk_array[reg],
+                                __x86_indirect_its_thunk_array[reg]);
+}
+
+/* Check if an indirect branch is at ITS-unsafe address */
+static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
+{
+       if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+               return false;
+
+       /* Indirect branch opcode is 2 or 3 bytes depending on reg */
+       addr += 1 + reg / 8;
+
+       /* Lower-half of the cacheline? */
+       return !(addr & 0x20);
+}
+#endif
+
 /*
  * Rewrite the compiler generated retpoline thunk calls.
  *
@@ -639,6 +669,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
                bytes[i++] = 0xe8; /* LFENCE */
        }
 
+#ifdef CONFIG_MITIGATION_ITS
+       /*
+        * Check if the address of last byte of emitted-indirect is in
+        * lower-half of the cacheline. Such branches need ITS mitigation.
+        */
+       if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))
+               return emit_its_trampoline(addr, insn, reg, bytes);
+#endif
+
        ret = emit_indirect(op, reg, bytes + i);
        if (ret < 0)
                return ret;

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 60eb8baa44d7b77ce94725dc8a845f7937ad55ba..27323cda907aef1ef1157a14d24a3c6fcf819b40 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -541,4 +541,10 @@ INIT_PER_CPU(irq_stack_backing_store);
                "SRSO function pair won't alias");
 #endif
 
+#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)
+. = ASSERT(__x86_indirect_its_thunk_rax & 0x20, "__x86_indirect_thunk_rax not in second half of cacheline");
+. = ASSERT(((__x86_indirect_its_thunk_rcx - __x86_indirect_its_thunk_rax) % 64) == 0, "Indirect thunks are not cacheline apart");
+. = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array");
+#endif
+
 #endif /* CONFIG_X86_64 */

diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index ffa51f392e17a3a923f8a72bb702fd994791f0e0..40d17f27499f3501a4d2929cbb8da9f556973d97 100644 (file)
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -360,6 +360,34 @@ SYM_FUNC_END(__x86_return_skl)
 
 #endif /* CONFIG_CALL_DEPTH_TRACKING */
 
+#ifdef CONFIG_MITIGATION_ITS
+
+.macro ITS_THUNK reg
+
+SYM_INNER_LABEL(__x86_indirect_its_thunk_\reg, SYM_L_GLOBAL)
+       UNWIND_HINT_UNDEFINED
+       ANNOTATE_NOENDBR
+       ANNOTATE_RETPOLINE_SAFE
+       jmp *%\reg
+       int3
+       .align 32, 0xcc         /* fill to the end of the line */
+       .skip  32, 0xcc         /* skip to the next upper half */
+.endm
+
+/* ITS mitigation requires thunks be aligned to upper half of cacheline */
+.align 64, 0xcc
+.skip 32, 0xcc
+SYM_CODE_START(__x86_indirect_its_thunk_array)
+
+#define GEN(reg) ITS_THUNK reg
+#include <asm/GEN-for-each-reg.h>
+#undef GEN
+
+       .align 64, 0xcc
+SYM_CODE_END(__x86_indirect_its_thunk_array)
+
+#endif
+
 /*
  * This function name is magical and is used by -mfunction-return=thunk-extern
  * for the compiler to generate JMPs to it.

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5a3ce74d132ba127a3ab68a3593c22056c8d7063..0d4218741e0d800d3fd42e101661f895edd1b226 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -472,7 +472,11 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
 {
        u8 *prog = *pprog;
 
-       if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
+       if (IS_ENABLED(CONFIG_MITIGATION_ITS) &&
+           cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {
+               OPTIMIZER_HIDE_VAR(reg);
+               emit_jump(&prog, &__x86_indirect_its_thunk_array[reg], ip);
+       } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
                EMIT_LFENCE();
                EMIT2(0xFF, 0xE0 + reg);
        } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {