x86/bhi: Add BHI stubs

Add an array of code thunks, to be called from the FineIBT preamble,
clobbering the first 'n' argument registers for speculative execution.

Notably the 0th entry will clobber no argument registers and will never
be used, it exists so the array can be naturally indexed, while the 7th
entry will clobber all the 6 argument registers and also RSP in order to
mess up stack based arguments.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kees Cook <kees@kernel.org>
Link: https://lore.kernel.org/r/20250224124200.717378681@infradead.org

diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h
index 7dd5ab239c87bdc48292cea07eb356db31bbc7c8..7c15c4bfe65f2919d467eeb5bed9eca12cfb020c 100644 (file)
--- a/arch/x86/include/asm/cfi.h
+++ b/arch/x86/include/asm/cfi.h
@@ -101,6 +101,10 @@ enum cfi_mode {
 
 extern enum cfi_mode cfi_mode;
 
+typedef u8 bhi_thunk[32];
+extern bhi_thunk __bhi_args[];
+extern bhi_thunk __bhi_args_end[];
+
 struct pt_regs;
 
 #ifdef CONFIG_CFI_CLANG

diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 8a59c61624c2f3f3580fce8231da86393241e7e9..f453507649d4854cc5c721d7bd04fbbf660e1eab 100644 (file)
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -66,5 +66,6 @@ endif
         lib-y += clear_page_64.o copy_page_64.o
         lib-y += memmove_64.o memset_64.o
         lib-y += copy_user_64.o copy_user_uncached_64.o
-       lib-y += cmpxchg16b_emu.o
+        lib-y += cmpxchg16b_emu.o
+        lib-y += bhi.o
 endif

diff --git a/arch/x86/lib/bhi.S b/arch/x86/lib/bhi.S
new file mode 100644 (file)
index 0000000..5889168
--- /dev/null
+++ b/arch/x86/lib/bhi.S
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/linkage.h>
+#include <asm/unwind_hints.h>
+#include <asm/nospec-branch.h>
+
+/*
+ * Notably, the FineIBT preamble calling these will have ZF set and r10 zero.
+ *
+ * The very last element is in fact larger than 32 bytes, but since its the
+ * last element, this does not matter,
+ *
+ * There are 2 #UD sites, located between 0,1-2,3 and 4,5-6,7 such that they
+ * can be reached using Jcc.d8, these elements (1 and 5) have sufficiently
+ * big alignment holes for this to not stagger the array.
+ */
+
+.pushsection .noinstr.text, "ax"
+
+       .align 32
+SYM_CODE_START(__bhi_args)
+
+#ifdef CONFIG_FINEIBT_BHI
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_0, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_1
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_1, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_1
+       cmovne %r10, %rdi
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 8
+       ANNOTATE_REACHABLE
+.Lud_1:        ud2
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_2, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_1
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_3, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_1
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       cmovne %r10, %rdx
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_4, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_2
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       cmovne %r10, %rdx
+       cmovne %r10, %rcx
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_5, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_2
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       cmovne %r10, %rdx
+       cmovne %r10, %rcx
+       cmovne %r10, %r8
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 8
+       ANNOTATE_REACHABLE
+.Lud_2:        ud2
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_6, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_2
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       cmovne %r10, %rdx
+       cmovne %r10, %rcx
+       cmovne %r10, %r8
+       cmovne %r10, %r9
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_7, SYM_L_LOCAL)
+       ANNOTATE_NOENDBR
+       UNWIND_HINT_FUNC
+       jne .Lud_2
+       cmovne %r10, %rdi
+       cmovne %r10, %rsi
+       cmovne %r10, %rdx
+       cmovne %r10, %rcx
+       cmovne %r10, %r8
+       cmovne %r10, %r9
+       cmovne %r10, %rsp
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+
+#endif /* CONFIG_FINEIBT_BHI */
+
+       .align 32
+SYM_INNER_LABEL(__bhi_args_end, SYM_L_GLOBAL)
+       ANNOTATE_NOENDBR
+       nop /* Work around toolchain+objtool quirk */
+SYM_CODE_END(__bhi_args)
+
+.popsection