x86/ibt: Optimize the fineibt-bhi arity 1 case

Saves a CALL to an out-of-line thunk for the common case of 1
argument.

Suggested-by: Scott Constable <scott.d.constable@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kees Cook <kees@kernel.org>
Link: https://lore.kernel.org/r/20250224124200.927885784@infradead.org

diff --git a/arch/x86/include/asm/ibt.h b/arch/x86/include/asm/ibt.h
index f0ca5c0b1a6b2842332c98a39a2ac493a4e721ac..9423a2967f506ab80bdd70c0ac0cc1c15e80dfc5 100644 (file)
--- a/arch/x86/include/asm/ibt.h
+++ b/arch/x86/include/asm/ibt.h
@@ -70,6 +70,10 @@ static inline bool __is_endbr(u32 val)
        if (val == gen_endbr_poison())
                return true;
 
+       /* See cfi_fineibt_bhi_preamble() */
+       if (IS_ENABLED(CONFIG_FINEIBT_BHI) && val == 0x001f0ff5)
+               return true;
+
        val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
        return val == gen_endbr();
 }

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index b8d65d5120665c684b21ac41ca045083ccaccbbe..32e4b801db991a1a80499571d2c38641ac751067 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1311,6 +1311,53 @@ static int cfi_rand_preamble(s32 *start, s32 *end)
        return 0;
 }
 
+static void cfi_fineibt_bhi_preamble(void *addr, int arity)
+{
+       if (!arity)
+               return;
+
+       if (!cfi_warn && arity == 1) {
+               /*
+                * Crazy scheme to allow arity-1 inline:
+                *
+                * __cfi_foo:
+                *  0: f3 0f 1e fa             endbr64
+                *  4: 41 81 <ea> 78 56 34 12  sub     0x12345678, %r10d
+                *  b: 49 0f 45 fa             cmovne  %r10, %rdi
+                *  f: 75 f5                   jne     __cfi_foo+6
+                * 11: 0f 1f 00                nopl    (%rax)
+                *
+                * Code that direct calls to foo()+0, decodes the tail end as:
+                *
+                * foo:
+                *  0: f5                      cmc
+                *  1: 0f 1f 00                nopl    (%rax)
+                *
+                * which clobbers CF, but does not affect anything ABI
+                * wise.
+                *
+                * Notably, this scheme is incompatible with permissive CFI
+                * because the CMOVcc is unconditional and RDI will have been
+                * clobbered.
+                */
+               const u8 magic[9] = {
+                       0x49, 0x0f, 0x45, 0xfa,
+                       0x75, 0xf5,
+                       BYTES_NOP3,
+               };
+
+               text_poke_early(addr + fineibt_preamble_bhi, magic, 9);
+
+               return;
+       }
+
+       text_poke_early(addr + fineibt_preamble_bhi,
+                       text_gen_insn(CALL_INSN_OPCODE,
+                                     addr + fineibt_preamble_bhi,
+                                     __bhi_args[arity]),
+                       CALL_INSN_SIZE);
+}
+
 static int cfi_rewrite_preamble(s32 *start, s32 *end)
 {
        s32 *s;
@@ -1341,14 +1388,8 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end)
                          "kCFI preamble has wrong register at: %pS %*ph\n",
                          addr, 5, addr);
 
-               if (!cfi_bhi || !arity)
-                       continue;
-
-               text_poke_early(addr + fineibt_preamble_bhi,
-                               text_gen_insn(CALL_INSN_OPCODE,
-                                             addr + fineibt_preamble_bhi,
-                                             __bhi_args[arity]),
-                               CALL_INSN_SIZE);
+               if (cfi_bhi)
+                       cfi_fineibt_bhi_preamble(addr, arity);
        }
 
        return 0;
@@ -1361,7 +1402,7 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end)
        for (s = start; s < end; s++) {
                void *addr = (void *)s + *s;
 
-               if (!is_endbr(addr + 16))
+               if (!exact_endbr(addr + 16))
                        continue;
 
                poison_endbr(addr + 16);