x86/its: Use dynamic thunks for indirect branches

commit 872df34d7c51a79523820ea6a14860398c639b87 upstream.

ITS mitigation moves the unsafe indirect branches to a safe thunk. This
could degrade the prediction accuracy as the source address of indirect
branches becomes same for different execution paths.

To improve the predictions, and hence the performance, assign a separate
thunk for each indirect callsite. This is also a defense-in-depth measure
to avoid indirect branches aliasing with each other.

As an example, 5000 dynamic thunks would utilize around 16 bits of the
address space, thereby gaining entropy. For a BTB that uses
32 bits for indexing, dynamic thunks could provide better prediction
accuracy over fixed thunks.

Have ITS thunks be variable sized and use EXECMEM_MODULE_TEXT such that
they are both more flexible (got to extend them later) and live in 2M TLBs,
just like kernel code, avoiding undue TLB pressure.

  [ pawan: CONFIG_EXECMEM and CONFIG_EXECMEM_ROX are not supported on
	   backport kernel, made changes to use module_alloc() and
	   set_memory_*() for dynamic thunks. ]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 2112939460856456314bee31f55b56aa91f5b4aa..4463cf38c16998ae61568665d763931616bf996d 100644 (file)
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -81,6 +81,16 @@ extern void apply_ibt_endbr(s32 *start, s32 *end);
 
 struct module;
 
+#ifdef CONFIG_MITIGATION_ITS
+extern void its_init_mod(struct module *mod);
+extern void its_fini_mod(struct module *mod);
+extern void its_free_mod(struct module *mod);
+#else /* CONFIG_MITIGATION_ITS */
+static inline void its_init_mod(struct module *mod) { }
+static inline void its_fini_mod(struct module *mod) { }
+static inline void its_free_mod(struct module *mod) { }
+#endif
+
 #if defined(CONFIG_RETHUNK) && defined(CONFIG_OBJTOOL)
 extern bool cpu_wants_rethunk(void);
 extern bool cpu_wants_rethunk_at(void *addr);

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index f46db81b55b0762fa73c783ee9e43aa61947c714..44bff34f7d10cb6868ad079ca1cb87e458d3f91b 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -18,6 +18,8 @@
 #include <linux/mmu_context.h>
 #include <linux/bsearch.h>
 #include <linux/sync_core.h>
+#include <linux/moduleloader.h>
+#include <linux/cleanup.h>
 #include <asm/text-patching.h>
 #include <asm/alternative.h>
 #include <asm/sections.h>
@@ -31,6 +33,7 @@
 #include <asm/paravirt.h>
 #include <asm/asm-prototypes.h>
 #include <asm/cfi.h>
+#include <asm/set_memory.h>
 
 int __read_mostly alternatives_patched;
 
@@ -399,6 +402,123 @@ static int emit_indirect(int op, int reg, u8 *bytes)
 
 #ifdef CONFIG_MITIGATION_ITS
 
+static struct module *its_mod;
+static void *its_page;
+static unsigned int its_offset;
+
+/* Initialize a thunk with the "jmp *reg; int3" instructions. */
+static void *its_init_thunk(void *thunk, int reg)
+{
+       u8 *bytes = thunk;
+       int i = 0;
+
+       if (reg >= 8) {
+               bytes[i++] = 0x41; /* REX.B prefix */
+               reg -= 8;
+       }
+       bytes[i++] = 0xff;
+       bytes[i++] = 0xe0 + reg; /* jmp *reg */
+       bytes[i++] = 0xcc;
+
+       return thunk;
+}
+
+void its_init_mod(struct module *mod)
+{
+       if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+               return;
+
+       mutex_lock(&text_mutex);
+       its_mod = mod;
+       its_page = NULL;
+}
+
+void its_fini_mod(struct module *mod)
+{
+       if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+               return;
+
+       WARN_ON_ONCE(its_mod != mod);
+
+       its_mod = NULL;
+       its_page = NULL;
+       mutex_unlock(&text_mutex);
+
+       for (int i = 0; i < mod->its_num_pages; i++) {
+               void *page = mod->its_page_array[i];
+               set_memory_ro((unsigned long)page, 1);
+               set_memory_x((unsigned long)page, 1);
+       }
+}
+
+void its_free_mod(struct module *mod)
+{
+       if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+               return;
+
+       for (int i = 0; i < mod->its_num_pages; i++) {
+               void *page = mod->its_page_array[i];
+               module_memfree(page);
+       }
+       kfree(mod->its_page_array);
+}
+
+DEFINE_FREE(its_execmem, void *, if (_T) module_memfree(_T));
+
+static void *its_alloc(void)
+{
+       void *page __free(its_execmem) = module_alloc(PAGE_SIZE);
+
+       if (!page)
+               return NULL;
+
+       if (its_mod) {
+               void *tmp = krealloc(its_mod->its_page_array,
+                                    (its_mod->its_num_pages+1) * sizeof(void *),
+                                    GFP_KERNEL);
+               if (!tmp)
+                       return NULL;
+
+               its_mod->its_page_array = tmp;
+               its_mod->its_page_array[its_mod->its_num_pages++] = page;
+       }
+
+       return no_free_ptr(page);
+}
+
+static void *its_allocate_thunk(int reg)
+{
+       int size = 3 + (reg / 8);
+       void *thunk;
+
+       if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {
+               its_page = its_alloc();
+               if (!its_page) {
+                       pr_err("ITS page allocation failed\n");
+                       return NULL;
+               }
+               memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE);
+               its_offset = 32;
+       }
+
+       /*
+        * If the indirect branch instruction will be in the lower half
+        * of a cacheline, then update the offset to reach the upper half.
+        */
+       if ((its_offset + size - 1) % 64 < 32)
+               its_offset = ((its_offset - 1) | 0x3F) + 33;
+
+       thunk = its_page + its_offset;
+       its_offset += size;
+
+       set_memory_rw((unsigned long)its_page, 1);
+       thunk = its_init_thunk(thunk, reg);
+       set_memory_ro((unsigned long)its_page, 1);
+       set_memory_x((unsigned long)its_page, 1);
+
+       return thunk;
+}
+
 static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
                             void *call_dest, void *jmp_dest)
 {
@@ -446,9 +566,13 @@ clang_jcc:
 
 static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
 {
-       return __emit_trampoline(addr, insn, bytes,
-                                __x86_indirect_its_thunk_array[reg],
-                                __x86_indirect_its_thunk_array[reg]);
+       u8 *thunk = __x86_indirect_its_thunk_array[reg];
+       u8 *tmp = its_allocate_thunk(reg);
+
+       if (tmp)
+               thunk = tmp;
+
+       return __emit_trampoline(addr, insn, bytes, thunk, thunk);
 }
 
 /* Check if an indirect branch is at ITS-unsafe address */

diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index c032edcd3d95e06941a4810e09ba18da1707b196..7728060b640cb0c9077e0f66d7ac547a8b8854cd 100644 (file)
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -285,10 +285,16 @@ int module_finalize(const Elf_Ehdr *hdr,
                void *pseg = (void *)para->sh_addr;
                apply_paravirt(pseg, pseg + para->sh_size);
        }
+
+       its_init_mod(me);
+
        if (retpolines) {
                void *rseg = (void *)retpolines->sh_addr;
                apply_retpolines(rseg, rseg + retpolines->sh_size);
        }
+
+       its_fini_mod(me);
+
        if (returns) {
                void *rseg = (void *)returns->sh_addr;
                apply_returns(rseg, rseg + returns->sh_size);
@@ -320,4 +326,5 @@ int module_finalize(const Elf_Ehdr *hdr,
 void module_arch_cleanup(struct module *mod)
 {
        alternatives_smp_module_del(mod);
+       its_free_mod(mod);
 }

diff --git a/include/linux/module.h b/include/linux/module.h
index 35876e89eb93fc939ef52949fd3d5c9c4064d0eb..41c4c472ed1750144bd135099fe53c9520f82a99 100644 (file)
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -536,6 +536,11 @@ struct module {
        atomic_t refcnt;
 #endif
 
+#ifdef CONFIG_MITIGATION_ITS
+       int its_num_pages;
+       void **its_page_array;
+#endif
+
 #ifdef CONFIG_CONSTRUCTORS
        /* Constructor functions. */
        ctor_fn_t *ctors;