From: Greg Kroah-Hartman Date: Mon, 12 May 2025 17:09:46 +0000 (+0200) Subject: 6.6-stable patches X-Git-Tag: v5.15.183~15 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=874c1c57a21b43721f2743d6fca5ae68832a7f7c;p=thirdparty%2Fkernel%2Fstable-queue.git 6.6-stable patches added patches: arm64-bpf-add-bhb-mitigation-to-the-epilogue-for-cbpf-programs.patch arm64-bpf-only-mitigate-cbpf-programs-loaded-by-unprivileged-users.patch arm64-insn-add-support-for-encoding-dsb.patch arm64-proton-pack-add-new-cpus-k-values-for-branch-mitigation.patch arm64-proton-pack-expose-whether-the-branchy-loop-k-value.patch arm64-proton-pack-expose-whether-the-platform-is-mitigated-by-firmware.patch documentation-x86-bugs-its-add-its-documentation.patch x86-bhi-do-not-set-bhi_dis_s-in-32-bit-mode.patch x86-bpf-add-ibhf-call-at-end-of-classic-bpf.patch x86-bpf-call-branch-history-clearing-sequence-on-exit.patch x86-ibt-keep-ibt-disabled-during-alternative-patching.patch x86-its-add-support-for-its-safe-indirect-thunk.patch x86-its-add-support-for-its-safe-return-thunk.patch x86-its-add-support-for-rsb-stuffing-mitigation.patch x86-its-add-vmexit-option-to-skip-mitigation-on-some-cpus.patch x86-its-align-rets-in-bhb-clear-sequence-to-avoid-thunking.patch x86-its-enable-indirect-target-selection-mitigation.patch x86-its-enumerate-indirect-target-selection-its-bug.patch x86-its-use-dynamic-thunks-for-indirect-branches.patch x86-speculation-add-a-conditional-cs-prefix-to-call_nospec.patch x86-speculation-remove-the-extra-ifdef-around-call_nospec.patch x86-speculation-simplify-and-make-call_nospec-consistent.patch --- diff --git a/queue-6.6/arm64-bpf-add-bhb-mitigation-to-the-epilogue-for-cbpf-programs.patch b/queue-6.6/arm64-bpf-add-bhb-mitigation-to-the-epilogue-for-cbpf-programs.patch new file mode 100644 index 0000000000..7c9dcc9407 --- /dev/null +++ b/queue-6.6/arm64-bpf-add-bhb-mitigation-to-the-epilogue-for-cbpf-programs.patch @@ -0,0 +1,149 @@ +From 6bd20b156eda62bca41d293c9218a28a0c5b81b6 Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Thu, 9 Dec 2021 15:13:24 +0000 +Subject: arm64: bpf: Add BHB mitigation to the epilogue for cBPF programs + +From: James Morse + +commit 0dfefc2ea2f29ced2416017d7e5b1253a54c2735 upstream. + +A malicious BPF program may manipulate the branch history to influence +what the hardware speculates will happen next. + +On exit from a BPF program, emit the BHB mititgation sequence. + +This is only applied for 'classic' cBPF programs that are loaded by +seccomp. + +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Acked-by: Daniel Borkmann +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/spectre.h | 1 + arch/arm64/kernel/proton-pack.c | 2 - + arch/arm64/net/bpf_jit_comp.c | 54 ++++++++++++++++++++++++++++++++++++--- + 3 files changed, 52 insertions(+), 5 deletions(-) + +--- a/arch/arm64/include/asm/spectre.h ++++ b/arch/arm64/include/asm/spectre.h +@@ -97,6 +97,7 @@ enum mitigation_state arm64_get_meltdown + + enum mitigation_state arm64_get_spectre_bhb_state(void); + bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); ++extern bool __nospectre_bhb; + u8 get_spectre_bhb_loop_value(void); + bool is_spectre_bhb_fw_mitigated(void); + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -1020,7 +1020,7 @@ static void this_cpu_set_vectors(enum ar + isb(); + } + +-static bool __read_mostly __nospectre_bhb; ++bool __read_mostly __nospectre_bhb; + static int __init parse_spectre_bhb_param(char *str) + { + __nospectre_bhb = true; +--- a/arch/arm64/net/bpf_jit_comp.c ++++ b/arch/arm64/net/bpf_jit_comp.c +@@ -7,6 +7,7 @@ + + #define pr_fmt(fmt) "bpf_jit: " fmt + ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -653,7 +655,48 @@ static void build_plt(struct jit_ctx *ct + plt->target = (u64)&dummy_tramp; + } + +-static void build_epilogue(struct jit_ctx *ctx) ++/* Clobbers BPF registers 1-4, aka x0-x3 */ ++static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx) ++{ ++ const u8 r1 = bpf2a64[BPF_REG_1]; /* aka x0 */ ++ u8 k = get_spectre_bhb_loop_value(); ++ ++ if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || ++ cpu_mitigations_off() || __nospectre_bhb || ++ arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) ++ return; ++ ++ if (supports_clearbhb(SCOPE_SYSTEM)) { ++ emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx); ++ return; ++ } ++ ++ if (k) { ++ emit_a64_mov_i64(r1, k, ctx); ++ emit(A64_B(1), ctx); ++ emit(A64_SUBS_I(true, r1, r1, 1), ctx); ++ emit(A64_B_(A64_COND_NE, -2), ctx); ++ emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_ISH), ctx); ++ emit(aarch64_insn_get_isb_value(), ctx); ++ } ++ ++ if (is_spectre_bhb_fw_mitigated()) { ++ emit(A64_ORR_I(false, r1, AARCH64_INSN_REG_ZR, ++ ARM_SMCCC_ARCH_WORKAROUND_3), ctx); ++ switch (arm_smccc_1_1_get_conduit()) { ++ case SMCCC_CONDUIT_HVC: ++ emit(aarch64_insn_get_hvc_value(), ctx); ++ break; ++ case SMCCC_CONDUIT_SMC: ++ emit(aarch64_insn_get_smc_value(), ctx); ++ break; ++ default: ++ pr_err_once("Firmware mitigation enabled with unknown conduit\n"); ++ } ++ } ++} ++ ++static void build_epilogue(struct jit_ctx *ctx, bool was_classic) + { + const u8 r0 = bpf2a64[BPF_REG_0]; + const u8 r6 = bpf2a64[BPF_REG_6]; +@@ -675,10 +718,13 @@ static void build_epilogue(struct jit_ct + emit(A64_POP(r8, r9, A64_SP), ctx); + emit(A64_POP(r6, r7, A64_SP), ctx); + ++ if (was_classic) ++ build_bhb_mitigation(ctx); ++ + /* Restore FP/LR registers */ + emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); + +- /* Set return value */ ++ /* Move the return value from bpf:r0 (aka x7) to x0 */ + emit(A64_MOV(1, A64_R(0), r0), ctx); + + /* Authenticate lr */ +@@ -1586,7 +1632,7 @@ struct bpf_prog *bpf_int_jit_compile(str + } + + ctx.epilogue_offset = ctx.idx; +- build_epilogue(&ctx); ++ build_epilogue(&ctx, was_classic); + build_plt(&ctx); + + extable_align = __alignof__(struct exception_table_entry); +@@ -1622,7 +1668,7 @@ skip_init_ctx: + goto out_off; + } + +- build_epilogue(&ctx); ++ build_epilogue(&ctx, was_classic); + build_plt(&ctx); + + /* 3. Extra pass to validate JITed code. */ diff --git a/queue-6.6/arm64-bpf-only-mitigate-cbpf-programs-loaded-by-unprivileged-users.patch b/queue-6.6/arm64-bpf-only-mitigate-cbpf-programs-loaded-by-unprivileged-users.patch new file mode 100644 index 0000000000..06b4473b7e --- /dev/null +++ b/queue-6.6/arm64-bpf-only-mitigate-cbpf-programs-loaded-by-unprivileged-users.patch @@ -0,0 +1,36 @@ +From 3006ccff7a6057ee564ca96cda666ee87999fd0e Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Tue, 29 Apr 2025 16:03:38 +0100 +Subject: arm64: bpf: Only mitigate cBPF programs loaded by unprivileged users + +From: James Morse + +commit f300769ead032513a68e4a02e806393402e626f8 upstream. + +Support for eBPF programs loaded by unprivileged users is typically +disabled. This means only cBPF programs need to be mitigated for BHB. + +In addition, only mitigate cBPF programs that were loaded by an +unprivileged user. Privileged users can also load the same program +via eBPF, making the mitigation pointless. + +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Acked-by: Daniel Borkmann +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/net/bpf_jit_comp.c | 3 +++ + 1 file changed, 3 insertions(+) + +--- a/arch/arm64/net/bpf_jit_comp.c ++++ b/arch/arm64/net/bpf_jit_comp.c +@@ -666,6 +666,9 @@ static void __maybe_unused build_bhb_mit + arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) + return; + ++ if (capable(CAP_SYS_ADMIN)) ++ return; ++ + if (supports_clearbhb(SCOPE_SYSTEM)) { + emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx); + return; diff --git a/queue-6.6/arm64-insn-add-support-for-encoding-dsb.patch b/queue-6.6/arm64-insn-add-support-for-encoding-dsb.patch new file mode 100644 index 0000000000..cdad7454da --- /dev/null +++ b/queue-6.6/arm64-insn-add-support-for-encoding-dsb.patch @@ -0,0 +1,126 @@ +From 1287a3cb0569a0464462496c2a7491b165e956de Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Thu, 9 Dec 2021 15:12:19 +0000 +Subject: arm64: insn: Add support for encoding DSB + +From: James Morse + +commit 63de8abd97ddb9b758bd8f915ecbd18e1f1a87a0 upstream. + +To generate code in the eBPF epilogue that uses the DSB instruction, +insn.c needs a heler to encode the type and domain. + +Re-use the crm encoding logic from the DMB instruction. + +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/insn.h | 1 + arch/arm64/lib/insn.c | 60 +++++++++++++++++++++++++----------------- + 2 files changed, 38 insertions(+), 23 deletions(-) + +--- a/arch/arm64/include/asm/insn.h ++++ b/arch/arm64/include/asm/insn.h +@@ -687,6 +687,7 @@ u32 aarch64_insn_gen_cas(enum aarch64_in + } + #endif + u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type); ++u32 aarch64_insn_gen_dsb(enum aarch64_insn_mb_type type); + + s32 aarch64_get_branch_offset(u32 insn); + u32 aarch64_set_branch_offset(u32 insn, s32 offset); +--- a/arch/arm64/lib/insn.c ++++ b/arch/arm64/lib/insn.c +@@ -5,6 +5,7 @@ + * + * Copyright (C) 2014-2016 Zi Shen Lim + */ ++#include + #include + #include + #include +@@ -1471,47 +1472,60 @@ u32 aarch64_insn_gen_extr(enum aarch64_i + return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm); + } + +-u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) ++static u32 __get_barrier_crm_val(enum aarch64_insn_mb_type type) + { +- u32 opt; +- u32 insn; +- + switch (type) { + case AARCH64_INSN_MB_SY: +- opt = 0xf; +- break; ++ return 0xf; + case AARCH64_INSN_MB_ST: +- opt = 0xe; +- break; ++ return 0xe; + case AARCH64_INSN_MB_LD: +- opt = 0xd; +- break; ++ return 0xd; + case AARCH64_INSN_MB_ISH: +- opt = 0xb; +- break; ++ return 0xb; + case AARCH64_INSN_MB_ISHST: +- opt = 0xa; +- break; ++ return 0xa; + case AARCH64_INSN_MB_ISHLD: +- opt = 0x9; +- break; ++ return 0x9; + case AARCH64_INSN_MB_NSH: +- opt = 0x7; +- break; ++ return 0x7; + case AARCH64_INSN_MB_NSHST: +- opt = 0x6; +- break; ++ return 0x6; + case AARCH64_INSN_MB_NSHLD: +- opt = 0x5; +- break; ++ return 0x5; + default: +- pr_err("%s: unknown dmb type %d\n", __func__, type); ++ pr_err("%s: unknown barrier type %d\n", __func__, type); + return AARCH64_BREAK_FAULT; + } ++} ++ ++u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) ++{ ++ u32 opt; ++ u32 insn; ++ ++ opt = __get_barrier_crm_val(type); ++ if (opt == AARCH64_BREAK_FAULT) ++ return AARCH64_BREAK_FAULT; + + insn = aarch64_insn_get_dmb_value(); + insn &= ~GENMASK(11, 8); + insn |= (opt << 8); + ++ return insn; ++} ++ ++u32 aarch64_insn_gen_dsb(enum aarch64_insn_mb_type type) ++{ ++ u32 opt, insn; ++ ++ opt = __get_barrier_crm_val(type); ++ if (opt == AARCH64_BREAK_FAULT) ++ return AARCH64_BREAK_FAULT; ++ ++ insn = aarch64_insn_get_dsb_base_value(); ++ insn &= ~GENMASK(11, 8); ++ insn |= (opt << 8); ++ + return insn; + } diff --git a/queue-6.6/arm64-proton-pack-add-new-cpus-k-values-for-branch-mitigation.patch b/queue-6.6/arm64-proton-pack-add-new-cpus-k-values-for-branch-mitigation.patch new file mode 100644 index 0000000000..1196efc6ae --- /dev/null +++ b/queue-6.6/arm64-proton-pack-add-new-cpus-k-values-for-branch-mitigation.patch @@ -0,0 +1,52 @@ +From 2cfff6f3aa2edbafcff2b256e520628fe14adbeb Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Mon, 12 Aug 2024 17:50:22 +0100 +Subject: arm64: proton-pack: Add new CPUs 'k' values for branch mitigation + +From: James Morse + +commit efe676a1a7554219eae0b0dcfe1e0cdcc9ef9aef upstream. + +Update the list of 'k' values for the branch mitigation from arm's +website. + +Add the values for Cortex-X1C. The MIDR_EL1 value can be found here: +https://developer.arm.com/documentation/101968/0002/Register-descriptions/AArch> + +Link: https://developer.arm.com/documentation/110280/2-0/?lang=en +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/cputype.h | 2 ++ + arch/arm64/kernel/proton-pack.c | 1 + + 2 files changed, 3 insertions(+) + +--- a/arch/arm64/include/asm/cputype.h ++++ b/arch/arm64/include/asm/cputype.h +@@ -81,6 +81,7 @@ + #define ARM_CPU_PART_CORTEX_A78AE 0xD42 + #define ARM_CPU_PART_CORTEX_X1 0xD44 + #define ARM_CPU_PART_CORTEX_A510 0xD46 ++#define ARM_CPU_PART_CORTEX_X1C 0xD4C + #define ARM_CPU_PART_CORTEX_A520 0xD80 + #define ARM_CPU_PART_CORTEX_A710 0xD47 + #define ARM_CPU_PART_CORTEX_A715 0xD4D +@@ -166,6 +167,7 @@ + #define MIDR_CORTEX_A78AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78AE) + #define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) + #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) ++#define MIDR_CORTEX_X1C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1C) + #define MIDR_CORTEX_A520 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520) + #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) + #define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715) +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -891,6 +891,7 @@ static u8 spectre_bhb_loop_affected(void + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), ++ MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), diff --git a/queue-6.6/arm64-proton-pack-expose-whether-the-branchy-loop-k-value.patch b/queue-6.6/arm64-proton-pack-expose-whether-the-branchy-loop-k-value.patch new file mode 100644 index 0000000000..03131cc9b7 --- /dev/null +++ b/queue-6.6/arm64-proton-pack-expose-whether-the-branchy-loop-k-value.patch @@ -0,0 +1,44 @@ +From 9bd1d1f18d249abe71a015191ab8e25938fe15f8 Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Tue, 29 Apr 2025 13:55:17 +0100 +Subject: arm64: proton-pack: Expose whether the branchy loop k value + +From: James Morse + +commit a1152be30a043d2d4dcb1683415f328bf3c51978 upstream. + +Add a helper to expose the k value of the branchy loop. This is needed +by the BPF JIT to generate the mitigation sequence in BPF programs. + +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/spectre.h | 1 + + arch/arm64/kernel/proton-pack.c | 5 +++++ + 2 files changed, 6 insertions(+) + +--- a/arch/arm64/include/asm/spectre.h ++++ b/arch/arm64/include/asm/spectre.h +@@ -97,6 +97,7 @@ enum mitigation_state arm64_get_meltdown + + enum mitigation_state arm64_get_spectre_bhb_state(void); + bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); ++u8 get_spectre_bhb_loop_value(void); + bool is_spectre_bhb_fw_mitigated(void); + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); + bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr); +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -998,6 +998,11 @@ bool is_spectre_bhb_affected(const struc + return true; + } + ++u8 get_spectre_bhb_loop_value(void) ++{ ++ return max_bhb_k; ++} ++ + static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) + { + const char *v = arm64_get_bp_hardening_vector(slot); diff --git a/queue-6.6/arm64-proton-pack-expose-whether-the-platform-is-mitigated-by-firmware.patch b/queue-6.6/arm64-proton-pack-expose-whether-the-platform-is-mitigated-by-firmware.patch new file mode 100644 index 0000000000..6390a0c14e --- /dev/null +++ b/queue-6.6/arm64-proton-pack-expose-whether-the-platform-is-mitigated-by-firmware.patch @@ -0,0 +1,57 @@ +From c4f26e7dc311d6c0dd91a2ad8896a77ce17a61f8 Mon Sep 17 00:00:00 2001 +From: James Morse +Date: Mon, 19 Aug 2024 14:15:53 +0100 +Subject: arm64: proton-pack: Expose whether the platform is mitigated by firmware + +From: James Morse + +commit e7956c92f396a44eeeb6eaf7a5b5e1ad24db6748 upstream. + +is_spectre_bhb_fw_affected() allows the caller to determine if the CPU +is known to need a firmware mitigation. CPUs are either on the list +of CPUs we know about, or firmware has been queried and reported that +the platform is affected - and mitigated by firmware. + +This helper is not useful to determine if the platform is mitigated +by firmware. A CPU could be on the know list, but the firmware may +not be implemented. Its affected but not mitigated. + +spectre_bhb_enable_mitigation() handles this distinction by checking +the firmware state before enabling the mitigation. + +Add a helper to expose this state. This will be used by the BPF JIT +to determine if calling firmware for a mitigation is necessary and +supported. + +Signed-off-by: James Morse +Reviewed-by: Catalin Marinas +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/spectre.h | 1 + + arch/arm64/kernel/proton-pack.c | 5 +++++ + 2 files changed, 6 insertions(+) + +--- a/arch/arm64/include/asm/spectre.h ++++ b/arch/arm64/include/asm/spectre.h +@@ -97,6 +97,7 @@ enum mitigation_state arm64_get_meltdown + + enum mitigation_state arm64_get_spectre_bhb_state(void); + bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); ++bool is_spectre_bhb_fw_mitigated(void); + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); + bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr); + +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -1093,6 +1093,11 @@ void spectre_bhb_enable_mitigation(const + update_mitigation_state(&spectre_bhb_state, state); + } + ++bool is_spectre_bhb_fw_mitigated(void) ++{ ++ return test_bit(BHB_FW, &system_bhb_mitigations); ++} ++ + /* Patched to NOP when enabled */ + void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, + __le32 *origptr, diff --git a/queue-6.6/documentation-x86-bugs-its-add-its-documentation.patch b/queue-6.6/documentation-x86-bugs-its-add-its-documentation.patch new file mode 100644 index 0000000000..cf7bae88d4 --- /dev/null +++ b/queue-6.6/documentation-x86-bugs-its-add-its-documentation.patch @@ -0,0 +1,200 @@ +From 05d4b0de7a368b3a216cef976a37f14de402926a Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 11 Apr 2025 15:36:38 -0700 +Subject: Documentation: x86/bugs/its: Add ITS documentation + +From: Pawan Gupta + +commit 1ac116ce6468670eeda39345a5585df308243dca upstream. + +Add the admin-guide for Indirect Target Selection (ITS). + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/admin-guide/hw-vuln/index.rst | 1 + Documentation/admin-guide/hw-vuln/indirect-target-selection.rst | 168 ++++++++++ + 2 files changed, 169 insertions(+) + create mode 100644 Documentation/admin-guide/hw-vuln/indirect-target-selection.rst + +--- a/Documentation/admin-guide/hw-vuln/index.rst ++++ b/Documentation/admin-guide/hw-vuln/index.rst +@@ -22,3 +22,4 @@ are configurable at compile, boot or run + srso + gather_data_sampling + reg-file-data-sampling ++ indirect-target-selection +--- /dev/null ++++ b/Documentation/admin-guide/hw-vuln/indirect-target-selection.rst +@@ -0,0 +1,168 @@ ++.. SPDX-License-Identifier: GPL-2.0 ++ ++Indirect Target Selection (ITS) ++=============================== ++ ++ITS is a vulnerability in some Intel CPUs that support Enhanced IBRS and were ++released before Alder Lake. ITS may allow an attacker to control the prediction ++of indirect branches and RETs located in the lower half of a cacheline. ++ ++ITS is assigned CVE-2024-28956 with a CVSS score of 4.7 (Medium). ++ ++Scope of Impact ++--------------- ++- **eIBRS Guest/Host Isolation**: Indirect branches in KVM/kernel may still be ++ predicted with unintended target corresponding to a branch in the guest. ++ ++- **Intra-Mode BTI**: In-kernel training such as through cBPF or other native ++ gadgets. ++ ++- **Indirect Branch Prediction Barrier (IBPB)**: After an IBPB, indirect ++ branches may still be predicted with targets corresponding to direct branches ++ executed prior to the IBPB. This is fixed by the IPU 2025.1 microcode, which ++ should be available via distro updates. Alternatively microcode can be ++ obtained from Intel's github repository [#f1]_. ++ ++Affected CPUs ++------------- ++Below is the list of ITS affected CPUs [#f2]_ [#f3]_: ++ ++ ======================== ============ ==================== =============== ++ Common name Family_Model eIBRS Intra-mode BTI ++ Guest/Host Isolation ++ ======================== ============ ==================== =============== ++ SKYLAKE_X (step >= 6) 06_55H Affected Affected ++ ICELAKE_X 06_6AH Not affected Affected ++ ICELAKE_D 06_6CH Not affected Affected ++ ICELAKE_L 06_7EH Not affected Affected ++ TIGERLAKE_L 06_8CH Not affected Affected ++ TIGERLAKE 06_8DH Not affected Affected ++ KABYLAKE_L (step >= 12) 06_8EH Affected Affected ++ KABYLAKE (step >= 13) 06_9EH Affected Affected ++ COMETLAKE 06_A5H Affected Affected ++ COMETLAKE_L 06_A6H Affected Affected ++ ROCKETLAKE 06_A7H Not affected Affected ++ ======================== ============ ==================== =============== ++ ++- All affected CPUs enumerate Enhanced IBRS feature. ++- IBPB isolation is affected on all ITS affected CPUs, and need a microcode ++ update for mitigation. ++- None of the affected CPUs enumerate BHI_CTRL which was introduced in Golden ++ Cove (Alder Lake and Sapphire Rapids). This can help guests to determine the ++ host's affected status. ++- Intel Atom CPUs are not affected by ITS. ++ ++Mitigation ++---------- ++As only the indirect branches and RETs that have their last byte of instruction ++in the lower half of the cacheline are vulnerable to ITS, the basic idea behind ++the mitigation is to not allow indirect branches in the lower half. ++ ++This is achieved by relying on existing retpoline support in the kernel, and in ++compilers. ITS-vulnerable retpoline sites are runtime patched to point to newly ++added ITS-safe thunks. These safe thunks consists of indirect branch in the ++second half of the cacheline. Not all retpoline sites are patched to thunks, if ++a retpoline site is evaluated to be ITS-safe, it is replaced with an inline ++indirect branch. ++ ++Dynamic thunks ++~~~~~~~~~~~~~~ ++From a dynamically allocated pool of safe-thunks, each vulnerable site is ++replaced with a new thunk, such that they get a unique address. This could ++improve the branch prediction accuracy. Also, it is a defense-in-depth measure ++against aliasing. ++ ++Note, for simplicity, indirect branches in eBPF programs are always replaced ++with a jump to a static thunk in __x86_indirect_its_thunk_array. If required, ++in future this can be changed to use dynamic thunks. ++ ++All vulnerable RETs are replaced with a static thunk, they do not use dynamic ++thunks. This is because RETs get their prediction from RSB mostly that does not ++depend on source address. RETs that underflow RSB may benefit from dynamic ++thunks. But, RETs significantly outnumber indirect branches, and any benefit ++from a unique source address could be outweighed by the increased icache ++footprint and iTLB pressure. ++ ++Retpoline ++~~~~~~~~~ ++Retpoline sequence also mitigates ITS-unsafe indirect branches. For this ++reason, when retpoline is enabled, ITS mitigation only relocates the RETs to ++safe thunks. Unless user requested the RSB-stuffing mitigation. ++ ++RSB Stuffing ++~~~~~~~~~~~~ ++RSB-stuffing via Call Depth Tracking is a mitigation for Retbleed RSB-underflow ++attacks. And it also mitigates RETs that are vulnerable to ITS. ++ ++Mitigation in guests ++^^^^^^^^^^^^^^^^^^^^ ++All guests deploy ITS mitigation by default, irrespective of eIBRS enumeration ++and Family/Model of the guest. This is because eIBRS feature could be hidden ++from a guest. One exception to this is when a guest enumerates BHI_DIS_S, which ++indicates that the guest is running on an unaffected host. ++ ++To prevent guests from unnecessarily deploying the mitigation on unaffected ++platforms, Intel has defined ITS_NO bit(62) in MSR IA32_ARCH_CAPABILITIES. When ++a guest sees this bit set, it should not enumerate the ITS bug. Note, this bit ++is not set by any hardware, but is **intended for VMMs to synthesize** it for ++guests as per the host's affected status. ++ ++Mitigation options ++^^^^^^^^^^^^^^^^^^ ++The ITS mitigation can be controlled using the "indirect_target_selection" ++kernel parameter. The available options are: ++ ++ ======== =================================================================== ++ on (default) Deploy the "Aligned branch/return thunks" mitigation. ++ If spectre_v2 mitigation enables retpoline, aligned-thunks are only ++ deployed for the affected RET instructions. Retpoline mitigates ++ indirect branches. ++ ++ off Disable ITS mitigation. ++ ++ vmexit Equivalent to "=on" if the CPU is affected by guest/host isolation ++ part of ITS. Otherwise, mitigation is not deployed. This option is ++ useful when host userspace is not in the threat model, and only ++ attacks from guest to host are considered. ++ ++ stuff Deploy RSB-fill mitigation when retpoline is also deployed. ++ Otherwise, deploy the default mitigation. When retpoline mitigation ++ is enabled, RSB-stuffing via Call-Depth-Tracking also mitigates ++ ITS. ++ ++ force Force the ITS bug and deploy the default mitigation. ++ ======== =================================================================== ++ ++Sysfs reporting ++--------------- ++ ++The sysfs file showing ITS mitigation status is: ++ ++ /sys/devices/system/cpu/vulnerabilities/indirect_target_selection ++ ++Note, microcode mitigation status is not reported in this file. ++ ++The possible values in this file are: ++ ++.. list-table:: ++ ++ * - Not affected ++ - The processor is not vulnerable. ++ * - Vulnerable ++ - System is vulnerable and no mitigation has been applied. ++ * - Vulnerable, KVM: Not affected ++ - System is vulnerable to intra-mode BTI, but not affected by eIBRS ++ guest/host isolation. ++ * - Mitigation: Aligned branch/return thunks ++ - The mitigation is enabled, affected indirect branches and RETs are ++ relocated to safe thunks. ++ * - Mitigation: Retpolines, Stuffing RSB ++ - The mitigation is enabled using retpoline and RSB stuffing. ++ ++References ++---------- ++.. [#f1] Microcode repository - https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files ++ ++.. [#f2] Affected Processors list - https://www.intel.com/content/www/us/en/developer/topic-technology/software-security-guidance/processors-affected-consolidated-product-cpu-model.html ++ ++.. [#f3] Affected Processors list (machine readable) - https://github.com/intel/Intel-affected-processor-list diff --git a/queue-6.6/series b/queue-6.6/series index 98de08ae0f..088168f58b 100644 --- a/queue-6.6/series +++ b/queue-6.6/series @@ -89,3 +89,25 @@ nvme-unblock-ctrl-state-transition-for-firmware-upda.patch do_umount-add-missing-barrier-before-refcount-checks.patch io_uring-always-arm-linked-timeouts-prior-to-issue.patch io_uring-ensure-deferred-completions-are-posted-for-multishot.patch +arm64-insn-add-support-for-encoding-dsb.patch +arm64-proton-pack-expose-whether-the-platform-is-mitigated-by-firmware.patch +arm64-proton-pack-expose-whether-the-branchy-loop-k-value.patch +arm64-bpf-add-bhb-mitigation-to-the-epilogue-for-cbpf-programs.patch +arm64-bpf-only-mitigate-cbpf-programs-loaded-by-unprivileged-users.patch +arm64-proton-pack-add-new-cpus-k-values-for-branch-mitigation.patch +x86-bpf-call-branch-history-clearing-sequence-on-exit.patch +x86-bpf-add-ibhf-call-at-end-of-classic-bpf.patch +x86-bhi-do-not-set-bhi_dis_s-in-32-bit-mode.patch +x86-speculation-simplify-and-make-call_nospec-consistent.patch +x86-speculation-add-a-conditional-cs-prefix-to-call_nospec.patch +x86-speculation-remove-the-extra-ifdef-around-call_nospec.patch +documentation-x86-bugs-its-add-its-documentation.patch +x86-its-enumerate-indirect-target-selection-its-bug.patch +x86-its-add-support-for-its-safe-indirect-thunk.patch +x86-its-add-support-for-its-safe-return-thunk.patch +x86-its-enable-indirect-target-selection-mitigation.patch +x86-its-add-vmexit-option-to-skip-mitigation-on-some-cpus.patch +x86-its-add-support-for-rsb-stuffing-mitigation.patch +x86-its-align-rets-in-bhb-clear-sequence-to-avoid-thunking.patch +x86-ibt-keep-ibt-disabled-during-alternative-patching.patch +x86-its-use-dynamic-thunks-for-indirect-branches.patch diff --git a/queue-6.6/x86-bhi-do-not-set-bhi_dis_s-in-32-bit-mode.patch b/queue-6.6/x86-bhi-do-not-set-bhi_dis_s-in-32-bit-mode.patch new file mode 100644 index 0000000000..3feb1876a7 --- /dev/null +++ b/queue-6.6/x86-bhi-do-not-set-bhi_dis_s-in-32-bit-mode.patch @@ -0,0 +1,66 @@ +From bfe82250d8ca8e4f15a11d4cb62dd28112246178 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Mon, 5 May 2025 14:35:12 -0700 +Subject: x86/bhi: Do not set BHI_DIS_S in 32-bit mode + +From: Pawan Gupta + +commit 073fdbe02c69c43fb7c0d547ec265c7747d4a646 upstream. + +With the possibility of intra-mode BHI via cBPF, complete mitigation for +BHI is to use IBHF (history fence) instruction with BHI_DIS_S set. Since +this new instruction is only available in 64-bit mode, setting BHI_DIS_S in +32-bit mode is only a partial mitigation. + +Do not set BHI_DIS_S in 32-bit mode so as to avoid reporting misleading +mitigated status. With this change IBHF won't be used in 32-bit mode, also +remove the CONFIG_X86_64 check from emit_spectre_bhb_barrier(). + +Suggested-by: Josh Poimboeuf +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/kernel/cpu/bugs.c | 5 +++-- + arch/x86/net/bpf_jit_comp.c | 5 +++-- + 2 files changed, 6 insertions(+), 4 deletions(-) + +--- a/arch/x86/kernel/cpu/bugs.c ++++ b/arch/x86/kernel/cpu/bugs.c +@@ -1677,10 +1677,11 @@ static void __init bhi_select_mitigation + return; + } + +- if (spec_ctrl_bhi_dis()) ++ if (!IS_ENABLED(CONFIG_X86_64)) + return; + +- if (!IS_ENABLED(CONFIG_X86_64)) ++ /* Mitigate in hardware if supported */ ++ if (spec_ctrl_bhi_dis()) + return; + + /* Mitigate KVM by default */ +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -1097,8 +1097,7 @@ static int emit_spectre_bhb_barrier(u8 * + /* Insert IBHF instruction */ + if ((cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP) && + cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) || +- (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_HW) && +- IS_ENABLED(CONFIG_X86_64))) { ++ cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_HW)) { + /* + * Add an Indirect Branch History Fence (IBHF). IBHF acts as a + * fence preventing branch history from before the fence from +@@ -1108,6 +1107,8 @@ static int emit_spectre_bhb_barrier(u8 * + * hardware that doesn't need or support it. The REP and REX.W + * prefixes are required by the microcode, and they also ensure + * that the NOP is unlikely to be used in existing code. ++ * ++ * IBHF is not a valid instruction in 32-bit mode. + */ + EMIT5(0xF3, 0x48, 0x0F, 0x1E, 0xF8); /* ibhf */ + } diff --git a/queue-6.6/x86-bpf-add-ibhf-call-at-end-of-classic-bpf.patch b/queue-6.6/x86-bpf-add-ibhf-call-at-end-of-classic-bpf.patch new file mode 100644 index 0000000000..d0958c7a06 --- /dev/null +++ b/queue-6.6/x86-bpf-add-ibhf-call-at-end-of-classic-bpf.patch @@ -0,0 +1,99 @@ +From 6324bd92082ce1d049654c5e915949f897e03586 Mon Sep 17 00:00:00 2001 +From: Daniel Sneddon +Date: Mon, 5 May 2025 14:35:12 -0700 +Subject: x86/bpf: Add IBHF call at end of classic BPF + +From: Daniel Sneddon + +commit 9f725eec8fc0b39bdc07dcc8897283c367c1a163 upstream. + +Classic BPF programs can be run by unprivileged users, allowing +unprivileged code to execute inside the kernel. Attackers can use this to +craft branch history in kernel mode that can influence the target of +indirect branches. + +BHI_DIS_S provides user-kernel isolation of branch history, but cBPF can be +used to bypass this protection by crafting branch history in kernel mode. +To stop intra-mode attacks via cBPF programs, Intel created a new +instruction Indirect Branch History Fence (IBHF). IBHF prevents the +predicted targets of subsequent indirect branches from being influenced by +branch history prior to the IBHF. IBHF is only effective while BHI_DIS_S is +enabled. + +Add the IBHF instruction to cBPF jitted code's exit path. Add the new fence +when the hardware mitigation is enabled (i.e., X86_FEATURE_CLEAR_BHB_HW is +set) or after the software sequence (X86_FEATURE_CLEAR_BHB_LOOP) is being +used in a virtual machine. Note that X86_FEATURE_CLEAR_BHB_HW and +X86_FEATURE_CLEAR_BHB_LOOP are mutually exclusive, so the JIT compiler will +only emit the new fence, not the SW sequence, when X86_FEATURE_CLEAR_BHB_HW +is set. + +Hardware that enumerates BHI_NO basically has BHI_DIS_S protections always +enabled, regardless of the value of BHI_DIS_S. Since BHI_DIS_S doesn't +protect against intra-mode attacks, enumerate BHI bug on BHI_NO hardware as +well. + +Signed-off-by: Daniel Sneddon +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Acked-by: Daniel Borkmann +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/kernel/cpu/common.c | 9 ++++++--- + arch/x86/net/bpf_jit_comp.c | 19 +++++++++++++++++++ + 2 files changed, 25 insertions(+), 3 deletions(-) + +--- a/arch/x86/kernel/cpu/common.c ++++ b/arch/x86/kernel/cpu/common.c +@@ -1476,9 +1476,12 @@ static void __init cpu_set_bug_bits(stru + if (vulnerable_to_rfds(x86_arch_cap_msr)) + setup_force_cpu_bug(X86_BUG_RFDS); + +- /* When virtualized, eIBRS could be hidden, assume vulnerable */ +- if (!(x86_arch_cap_msr & ARCH_CAP_BHI_NO) && +- !cpu_matches(cpu_vuln_whitelist, NO_BHI) && ++ /* ++ * Intel parts with eIBRS are vulnerable to BHI attacks. Parts with ++ * BHI_NO still need to use the BHI mitigation to prevent Intra-mode ++ * attacks. When virtualized, eIBRS could be hidden, assume vulnerable. ++ */ ++ if (!cpu_matches(cpu_vuln_whitelist, NO_BHI) && + (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) || + boot_cpu_has(X86_FEATURE_HYPERVISOR))) + setup_force_cpu_bug(X86_BUG_BHI); +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -37,6 +37,8 @@ static u8 *emit_code(u8 *ptr, u32 bytes, + #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) + #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) + #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) ++#define EMIT5(b1, b2, b3, b4, b5) \ ++ do { EMIT1(b1); EMIT4(b2, b3, b4, b5); } while (0) + + #define EMIT1_off32(b1, off) \ + do { EMIT1(b1); EMIT(off, 4); } while (0) +@@ -1092,6 +1094,23 @@ static int emit_spectre_bhb_barrier(u8 * + EMIT1(0x59); /* pop rcx */ + EMIT1(0x58); /* pop rax */ + } ++ /* Insert IBHF instruction */ ++ if ((cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP) && ++ cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) || ++ (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_HW) && ++ IS_ENABLED(CONFIG_X86_64))) { ++ /* ++ * Add an Indirect Branch History Fence (IBHF). IBHF acts as a ++ * fence preventing branch history from before the fence from ++ * affecting indirect branches after the fence. This is ++ * specifically used in cBPF jitted code to prevent Intra-mode ++ * BHI attacks. The IBHF instruction is designed to be a NOP on ++ * hardware that doesn't need or support it. The REP and REX.W ++ * prefixes are required by the microcode, and they also ensure ++ * that the NOP is unlikely to be used in existing code. ++ */ ++ EMIT5(0xF3, 0x48, 0x0F, 0x1E, 0xF8); /* ibhf */ ++ } + *pprog = prog; + return 0; + } diff --git a/queue-6.6/x86-bpf-call-branch-history-clearing-sequence-on-exit.patch b/queue-6.6/x86-bpf-call-branch-history-clearing-sequence-on-exit.patch new file mode 100644 index 0000000000..3f1119cc9a --- /dev/null +++ b/queue-6.6/x86-bpf-call-branch-history-clearing-sequence-on-exit.patch @@ -0,0 +1,80 @@ +From 24a4d1a58f919f067a02efbc76d1fe1710c4ea72 Mon Sep 17 00:00:00 2001 +From: Daniel Sneddon +Date: Mon, 5 May 2025 14:35:12 -0700 +Subject: x86/bpf: Call branch history clearing sequence on exit + +From: Daniel Sneddon + +commit d4e89d212d401672e9cdfe825d947ee3a9fbe3f5 upstream. + +Classic BPF programs have been identified as potential vectors for +intra-mode Branch Target Injection (BTI) attacks. Classic BPF programs can +be run by unprivileged users. They allow unprivileged code to execute +inside the kernel. Attackers can use unprivileged cBPF to craft branch +history in kernel mode that can influence the target of indirect branches. + +Introduce a branch history buffer (BHB) clearing sequence during the JIT +compilation of classic BPF programs. The clearing sequence is the same as +is used in previous mitigations to protect syscalls. Since eBPF programs +already have their own mitigations in place, only insert the call on +classic programs that aren't run by privileged users. + +Signed-off-by: Daniel Sneddon +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Acked-by: Daniel Borkmann +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/net/bpf_jit_comp.c | 33 +++++++++++++++++++++++++++++++++ + 1 file changed, 33 insertions(+) + +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -1072,6 +1072,30 @@ static void emit_shiftx(u8 **pprog, u32 + #define RESTORE_TAIL_CALL_CNT(stack) \ + EMIT3_off32(0x48, 0x8B, 0x85, -round_up(stack, 8) - 8) + ++static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, ++ struct bpf_prog *bpf_prog) ++{ ++ u8 *prog = *pprog; ++ u8 *func; ++ ++ if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP)) { ++ /* The clearing sequence clobbers eax and ecx. */ ++ EMIT1(0x50); /* push rax */ ++ EMIT1(0x51); /* push rcx */ ++ ip += 2; ++ ++ func = (u8 *)clear_bhb_loop; ++ ip += x86_call_depth_emit_accounting(&prog, func); ++ ++ if (emit_call(&prog, func, ip)) ++ return -EINVAL; ++ EMIT1(0x59); /* pop rcx */ ++ EMIT1(0x58); /* pop rax */ ++ } ++ *pprog = prog; ++ return 0; ++} ++ + static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, + int oldproglen, struct jit_context *ctx, bool jmp_padding) + { +@@ -1945,6 +1969,15 @@ emit_jmp: + seen_exit = true; + /* Update cleanup_addr */ + ctx->cleanup_addr = proglen; ++ ++ if (bpf_prog_was_classic(bpf_prog) && ++ !capable(CAP_SYS_ADMIN)) { ++ u8 *ip = image + addrs[i - 1]; ++ ++ if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog)) ++ return -EINVAL; ++ } ++ + pop_callee_regs(&prog, callee_regs_used); + EMIT1(0xC9); /* leave */ + emit_return(&prog, image + addrs[i - 1] + (prog - temp)); diff --git a/queue-6.6/x86-ibt-keep-ibt-disabled-during-alternative-patching.patch b/queue-6.6/x86-ibt-keep-ibt-disabled-during-alternative-patching.patch new file mode 100644 index 0000000000..cc9b038354 --- /dev/null +++ b/queue-6.6/x86-ibt-keep-ibt-disabled-during-alternative-patching.patch @@ -0,0 +1,87 @@ +From f3d6a17a1a2844a42b95f9b480e438f19dddc734 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Sat, 3 May 2025 09:46:31 -0700 +Subject: x86/ibt: Keep IBT disabled during alternative patching + +From: Pawan Gupta + +commit ebebe30794d38c51f71fe4951ba6af4159d9837d upstream. + +cfi_rewrite_callers() updates the fineIBT hash matching at the caller side, +but except for paranoid-mode it relies on apply_retpoline() and friends for +any ENDBR relocation. This could temporarily cause an indirect branch to +land on a poisoned ENDBR. + +For instance, with para-virtualization enabled, a simple wrmsrl() could +have an indirect branch pointing to native_write_msr() who's ENDBR has been +relocated due to fineIBT: + +: + push %rbp + mov %rsp,%rbp + mov %esi,%eax + mov %rsi,%rdx + shr $0x20,%rdx + mov %edi,%edi + mov %rax,%rsi + call *0x21e65d0(%rip) # + ^^^^^^^^^^^^^^^^^^^^^^^ + +Such an indirect call during the alternative patching could #CP if the +caller is not *yet* adjusted for the new target ENDBR. To prevent a false + #CP, keep CET-IBT disabled until all callers are patched. + +Patching during the module load does not need to be guarded by IBT-disable +because the module code is not executed until the patching is complete. + + [ pawan: Since apply_paravirt() happens before __apply_fineibt() + relocates the ENDBR, pv_ops in the example above is not relevant. + It is still safer to keep this commit because missing an ENDBR + means an oops. ] + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/kernel/alternative.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -30,6 +30,7 @@ + #include + #include + #include ++#include + + int __read_mostly alternatives_patched; + +@@ -1629,6 +1630,8 @@ static noinline void __init alt_reloc_se + + void __init alternative_instructions(void) + { ++ u64 ibt; ++ + int3_selftest(); + + /* +@@ -1666,6 +1669,9 @@ void __init alternative_instructions(voi + */ + paravirt_set_cap(); + ++ /* Keep CET-IBT disabled until caller/callee are patched */ ++ ibt = ibt_save(/*disable*/ true); ++ + /* + * First patch paravirt functions, such that we overwrite the indirect + * call with the direct call. +@@ -1699,6 +1705,8 @@ void __init alternative_instructions(voi + */ + apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); + ++ ibt_restore(ibt); ++ + #ifdef CONFIG_SMP + /* Patch to UP if other cpus not imminent. */ + if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { diff --git a/queue-6.6/x86-its-add-support-for-its-safe-indirect-thunk.patch b/queue-6.6/x86-its-add-support-for-its-safe-indirect-thunk.patch new file mode 100644 index 0000000000..4d7321a9ba --- /dev/null +++ b/queue-6.6/x86-its-add-support-for-its-safe-indirect-thunk.patch @@ -0,0 +1,235 @@ +From e2d9c43ced0a36adcb8117f0e3a1ad5ce5ee8a02 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 21 Jun 2024 21:17:21 -0700 +Subject: x86/its: Add support for ITS-safe indirect thunk + +From: Pawan Gupta + +commit 8754e67ad4ac692c67ff1f99c0d07156f04ae40c upstream. + +Due to ITS, indirect branches in the lower half of a cacheline may be +vulnerable to branch target injection attack. + +Introduce ITS-safe thunks to patch indirect branches in the lower half of +cacheline with the thunk. Also thunk any eBPF generated indirect branches +in emit_indirect_jump(). + +Below category of indirect branches are not mitigated: + +- Indirect branches in the .init section are not mitigated because they are + discarded after boot. +- Indirect branches that are explicitly marked retpoline-safe. + +Note that retpoline also mitigates the indirect branches against ITS. This +is because the retpoline sequence fills an RSB entry before RET, and it +does not suffer from RSB-underflow part of the ITS. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/Kconfig | 11 ++++++++ + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/nospec-branch.h | 4 +++ + arch/x86/kernel/alternative.c | 45 ++++++++++++++++++++++++++++++++--- + arch/x86/kernel/vmlinux.lds.S | 6 ++++ + arch/x86/lib/retpoline.S | 28 +++++++++++++++++++++ + arch/x86/net/bpf_jit_comp.c | 6 +++- + 7 files changed, 97 insertions(+), 4 deletions(-) + +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -2610,6 +2610,17 @@ config MITIGATION_SPECTRE_BHI + indirect branches. + See + ++config MITIGATION_ITS ++ bool "Enable Indirect Target Selection mitigation" ++ depends on CPU_SUP_INTEL && X86_64 ++ depends on RETPOLINE && RETHUNK ++ default y ++ help ++ Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in ++ BPU on some Intel CPUs that may allow Spectre V2 style attacks. If ++ disabled, mitigation cannot be enabled via cmdline. ++ See ++ + endif + + config ARCH_HAS_ADD_PAGES +--- a/arch/x86/include/asm/cpufeatures.h ++++ b/arch/x86/include/asm/cpufeatures.h +@@ -468,6 +468,7 @@ + #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* "" BHI_DIS_S HW control available */ + #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* "" BHI_DIS_S HW control enabled */ + #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */ ++#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 5) /* "" Use thunk for indirect branches in lower half of cacheline */ + + /* + * BUG word(s) +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -364,10 +364,14 @@ + ".long 999b\n\t" \ + ".popsection\n\t" + ++#define ITS_THUNK_SIZE 64 ++ + typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; ++typedef u8 its_thunk_t[ITS_THUNK_SIZE]; + extern retpoline_thunk_t __x86_indirect_thunk_array[]; + extern retpoline_thunk_t __x86_indirect_call_thunk_array[]; + extern retpoline_thunk_t __x86_indirect_jump_thunk_array[]; ++extern its_thunk_t __x86_indirect_its_thunk_array[]; + + #ifdef CONFIG_RETHUNK + extern void __x86_return_thunk(void); +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -521,7 +521,8 @@ static int emit_indirect(int op, int reg + return i; + } + +-static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) ++static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes, ++ void *call_dest, void *jmp_dest) + { + u8 op = insn->opcode.bytes[0]; + int i = 0; +@@ -542,7 +543,7 @@ static int emit_call_track_retpoline(voi + switch (op) { + case CALL_INSN_OPCODE: + __text_gen_insn(bytes+i, op, addr+i, +- __x86_indirect_call_thunk_array[reg], ++ call_dest, + CALL_INSN_SIZE); + i += CALL_INSN_SIZE; + break; +@@ -550,7 +551,7 @@ static int emit_call_track_retpoline(voi + case JMP32_INSN_OPCODE: + clang_jcc: + __text_gen_insn(bytes+i, op, addr+i, +- __x86_indirect_jump_thunk_array[reg], ++ jmp_dest, + JMP32_INSN_SIZE); + i += JMP32_INSN_SIZE; + break; +@@ -565,6 +566,35 @@ clang_jcc: + return i; + } + ++static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) ++{ ++ return __emit_trampoline(addr, insn, bytes, ++ __x86_indirect_call_thunk_array[reg], ++ __x86_indirect_jump_thunk_array[reg]); ++} ++ ++#ifdef CONFIG_MITIGATION_ITS ++static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) ++{ ++ return __emit_trampoline(addr, insn, bytes, ++ __x86_indirect_its_thunk_array[reg], ++ __x86_indirect_its_thunk_array[reg]); ++} ++ ++/* Check if an indirect branch is at ITS-unsafe address */ ++static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) ++{ ++ if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) ++ return false; ++ ++ /* Indirect branch opcode is 2 or 3 bytes depending on reg */ ++ addr += 1 + reg / 8; ++ ++ /* Lower-half of the cacheline? */ ++ return !(addr & 0x20); ++} ++#endif ++ + /* + * Rewrite the compiler generated retpoline thunk calls. + * +@@ -639,6 +669,15 @@ static int patch_retpoline(void *addr, s + bytes[i++] = 0xe8; /* LFENCE */ + } + ++#ifdef CONFIG_MITIGATION_ITS ++ /* ++ * Check if the address of last byte of emitted-indirect is in ++ * lower-half of the cacheline. Such branches need ITS mitigation. ++ */ ++ if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg)) ++ return emit_its_trampoline(addr, insn, reg, bytes); ++#endif ++ + ret = emit_indirect(op, reg, bytes + i); + if (ret < 0) + return ret; +--- a/arch/x86/kernel/vmlinux.lds.S ++++ b/arch/x86/kernel/vmlinux.lds.S +@@ -541,4 +541,10 @@ INIT_PER_CPU(irq_stack_backing_store); + "SRSO function pair won't alias"); + #endif + ++#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B) ++. = ASSERT(__x86_indirect_its_thunk_rax & 0x20, "__x86_indirect_thunk_rax not in second half of cacheline"); ++. = ASSERT(((__x86_indirect_its_thunk_rcx - __x86_indirect_its_thunk_rax) % 64) == 0, "Indirect thunks are not cacheline apart"); ++. = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array"); ++#endif ++ + #endif /* CONFIG_X86_64 */ +--- a/arch/x86/lib/retpoline.S ++++ b/arch/x86/lib/retpoline.S +@@ -360,6 +360,34 @@ SYM_FUNC_END(__x86_return_skl) + + #endif /* CONFIG_CALL_DEPTH_TRACKING */ + ++#ifdef CONFIG_MITIGATION_ITS ++ ++.macro ITS_THUNK reg ++ ++SYM_INNER_LABEL(__x86_indirect_its_thunk_\reg, SYM_L_GLOBAL) ++ UNWIND_HINT_UNDEFINED ++ ANNOTATE_NOENDBR ++ ANNOTATE_RETPOLINE_SAFE ++ jmp *%\reg ++ int3 ++ .align 32, 0xcc /* fill to the end of the line */ ++ .skip 32, 0xcc /* skip to the next upper half */ ++.endm ++ ++/* ITS mitigation requires thunks be aligned to upper half of cacheline */ ++.align 64, 0xcc ++.skip 32, 0xcc ++SYM_CODE_START(__x86_indirect_its_thunk_array) ++ ++#define GEN(reg) ITS_THUNK reg ++#include ++#undef GEN ++ ++ .align 64, 0xcc ++SYM_CODE_END(__x86_indirect_its_thunk_array) ++ ++#endif ++ + /* + * This function name is magical and is used by -mfunction-return=thunk-extern + * for the compiler to generate JMPs to it. +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -472,7 +472,11 @@ static void emit_indirect_jump(u8 **ppro + { + u8 *prog = *pprog; + +- if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { ++ if (IS_ENABLED(CONFIG_MITIGATION_ITS) && ++ cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { ++ OPTIMIZER_HIDE_VAR(reg); ++ emit_jump(&prog, &__x86_indirect_its_thunk_array[reg], ip); ++ } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + EMIT2(0xFF, 0xE0 + reg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { diff --git a/queue-6.6/x86-its-add-support-for-its-safe-return-thunk.patch b/queue-6.6/x86-its-add-support-for-its-safe-return-thunk.patch new file mode 100644 index 0000000000..65249ee55a --- /dev/null +++ b/queue-6.6/x86-its-add-support-for-its-safe-return-thunk.patch @@ -0,0 +1,194 @@ +From 0f3fc4bb8ce2e83d1116a503d7d284a16114f4b9 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 21 Jun 2024 21:17:21 -0700 +Subject: x86/its: Add support for ITS-safe return thunk + +From: Pawan Gupta + +commit a75bf27fe41abe658c53276a0c486c4bf9adecfc upstream. + +RETs in the lower half of cacheline may be affected by ITS bug, +specifically when the RSB-underflows. Use ITS-safe return thunk for such +RETs. + +RETs that are not patched: + +- RET in retpoline sequence does not need to be patched, because the + sequence itself fills an RSB before RET. +- RET in Call Depth Tracking (CDT) thunks __x86_indirect_{call|jump}_thunk + and call_depth_return_thunk are not patched because CDT by design + prevents RSB-underflow. +- RETs in .init section are not reachable after init. +- RETs that are explicitly marked safe with ANNOTATE_UNRET_SAFE. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/alternative.h | 14 ++++++++++++++ + arch/x86/include/asm/nospec-branch.h | 6 ++++++ + arch/x86/kernel/alternative.c | 19 +++++++++++++++++-- + arch/x86/kernel/ftrace.c | 2 +- + arch/x86/kernel/static_call.c | 4 ++-- + arch/x86/kernel/vmlinux.lds.S | 4 ++++ + arch/x86/lib/retpoline.S | 13 ++++++++++++- + arch/x86/net/bpf_jit_comp.c | 2 +- + 8 files changed, 57 insertions(+), 7 deletions(-) + +--- a/arch/x86/include/asm/alternative.h ++++ b/arch/x86/include/asm/alternative.h +@@ -130,6 +130,20 @@ static __always_inline int x86_call_dept + } + #endif + ++#if defined(CONFIG_RETHUNK) && defined(CONFIG_OBJTOOL) ++extern bool cpu_wants_rethunk(void); ++extern bool cpu_wants_rethunk_at(void *addr); ++#else ++static __always_inline bool cpu_wants_rethunk(void) ++{ ++ return false; ++} ++static __always_inline bool cpu_wants_rethunk_at(void *addr) ++{ ++ return false; ++} ++#endif ++ + #ifdef CONFIG_SMP + extern void alternatives_smp_module_add(struct module *mod, char *name, + void *locks, void *locks_end, +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -395,6 +395,12 @@ static inline void srso_return_thunk(voi + static inline void srso_alias_return_thunk(void) {} + #endif + ++#ifdef CONFIG_MITIGATION_ITS ++extern void its_return_thunk(void); ++#else ++static inline void its_return_thunk(void) {} ++#endif ++ + extern void retbleed_return_thunk(void); + extern void srso_return_thunk(void); + extern void srso_alias_return_thunk(void); +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -749,6 +749,21 @@ void __init_or_module noinline apply_ret + + #ifdef CONFIG_RETHUNK + ++bool cpu_wants_rethunk(void) ++{ ++ return cpu_feature_enabled(X86_FEATURE_RETHUNK); ++} ++ ++bool cpu_wants_rethunk_at(void *addr) ++{ ++ if (!cpu_feature_enabled(X86_FEATURE_RETHUNK)) ++ return false; ++ if (x86_return_thunk != its_return_thunk) ++ return true; ++ ++ return !((unsigned long)addr & 0x20); ++} ++ + /* + * Rewrite the compiler generated return thunk tail-calls. + * +@@ -765,7 +780,7 @@ static int patch_return(void *addr, stru + int i = 0; + + /* Patch the custom return thunks... */ +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) { ++ if (cpu_wants_rethunk_at(addr)) { + i = JMP32_INSN_SIZE; + __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i); + } else { +@@ -782,7 +797,7 @@ void __init_or_module noinline apply_ret + { + s32 *s; + +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) ++ if (cpu_wants_rethunk()) + static_call_force_reinit(); + + for (s = start; s < end; s++) { +--- a/arch/x86/kernel/ftrace.c ++++ b/arch/x86/kernel/ftrace.c +@@ -363,7 +363,7 @@ create_trampoline(struct ftrace_ops *ops + goto fail; + + ip = trampoline + size; +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) ++ if (cpu_wants_rethunk_at(ip)) + __text_gen_insn(ip, JMP32_INSN_OPCODE, ip, x86_return_thunk, JMP32_INSN_SIZE); + else + memcpy(ip, retq, sizeof(retq)); +--- a/arch/x86/kernel/static_call.c ++++ b/arch/x86/kernel/static_call.c +@@ -81,7 +81,7 @@ static void __ref __static_call_transfor + break; + + case RET: +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) ++ if (cpu_wants_rethunk_at(insn)) + code = text_gen_insn(JMP32_INSN_OPCODE, insn, x86_return_thunk); + else + code = &retinsn; +@@ -90,7 +90,7 @@ static void __ref __static_call_transfor + case JCC: + if (!func) { + func = __static_call_return; +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) ++ if (cpu_wants_rethunk()) + func = x86_return_thunk; + } + +--- a/arch/x86/kernel/vmlinux.lds.S ++++ b/arch/x86/kernel/vmlinux.lds.S +@@ -547,4 +547,8 @@ INIT_PER_CPU(irq_stack_backing_store); + . = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array"); + #endif + ++#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B) ++. = ASSERT(its_return_thunk & 0x20, "its_return_thunk not in second half of cacheline"); ++#endif ++ + #endif /* CONFIG_X86_64 */ +--- a/arch/x86/lib/retpoline.S ++++ b/arch/x86/lib/retpoline.S +@@ -386,7 +386,18 @@ SYM_CODE_START(__x86_indirect_its_thunk_ + .align 64, 0xcc + SYM_CODE_END(__x86_indirect_its_thunk_array) + +-#endif ++.align 64, 0xcc ++.skip 32, 0xcc ++SYM_CODE_START(its_return_thunk) ++ UNWIND_HINT_FUNC ++ ANNOTATE_NOENDBR ++ ANNOTATE_UNRET_SAFE ++ ret ++ int3 ++SYM_CODE_END(its_return_thunk) ++EXPORT_SYMBOL(its_return_thunk) ++ ++#endif /* CONFIG_MITIGATION_ITS */ + + /* + * This function name is magical and is used by -mfunction-return=thunk-extern +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -498,7 +498,7 @@ static void emit_return(u8 **pprog, u8 * + { + u8 *prog = *pprog; + +- if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) { ++ if (cpu_wants_rethunk()) { + emit_jump(&prog, x86_return_thunk, ip); + } else { + EMIT1(0xC3); /* ret */ diff --git a/queue-6.6/x86-its-add-support-for-rsb-stuffing-mitigation.patch b/queue-6.6/x86-its-add-support-for-rsb-stuffing-mitigation.patch new file mode 100644 index 0000000000..e41ecef441 --- /dev/null +++ b/queue-6.6/x86-its-add-support-for-rsb-stuffing-mitigation.patch @@ -0,0 +1,89 @@ +From 5051f034e1933350075a63d6d63de6d49da861d2 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Mon, 2 Dec 2024 12:07:08 -0800 +Subject: x86/its: Add support for RSB stuffing mitigation + +From: Pawan Gupta + +commit facd226f7e0c8ca936ac114aba43cb3e8b94e41e upstream. + +When retpoline mitigation is enabled for spectre-v2, enabling +call-depth-tracking and RSB stuffing also mitigates ITS. Add cmdline option +indirect_target_selection=stuff to allow enabling RSB stuffing mitigation. + +When retpoline mitigation is not enabled, =stuff option is ignored, and +default mitigation for ITS is deployed. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/admin-guide/kernel-parameters.txt | 3 +++ + arch/x86/kernel/cpu/bugs.c | 21 +++++++++++++++++++++ + 2 files changed, 24 insertions(+) + +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -2070,6 +2070,9 @@ + mitigation. + vmexit: Only deploy mitigation if CPU is affected by + guest/host isolation part of ITS. ++ stuff: Deploy RSB-fill mitigation when retpoline is ++ also deployed. Otherwise, deploy the default ++ mitigation. + + For details see: + Documentation/admin-guide/hw-vuln/indirect-target-selection.rst +--- a/arch/x86/kernel/cpu/bugs.c ++++ b/arch/x86/kernel/cpu/bugs.c +@@ -1189,6 +1189,7 @@ enum its_mitigation_cmd { + ITS_CMD_OFF, + ITS_CMD_ON, + ITS_CMD_VMEXIT, ++ ITS_CMD_RSB_STUFF, + }; + + enum its_mitigation { +@@ -1229,6 +1230,8 @@ static int __init its_parse_cmdline(char + setup_force_cpu_bug(X86_BUG_ITS); + } else if (!strcmp(str, "vmexit")) { + its_cmd = ITS_CMD_VMEXIT; ++ } else if (!strcmp(str, "stuff")) { ++ its_cmd = ITS_CMD_RSB_STUFF; + } else { + pr_err("Ignoring unknown indirect_target_selection option (%s).", str); + } +@@ -1279,6 +1282,12 @@ static void __init its_select_mitigation + goto out; + } + ++ if (cmd == ITS_CMD_RSB_STUFF && ++ (!boot_cpu_has(X86_FEATURE_RETPOLINE) || !IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING))) { ++ pr_err("RSB stuff mitigation not supported, using default\n"); ++ cmd = ITS_CMD_ON; ++ } ++ + switch (cmd) { + case ITS_CMD_OFF: + its_mitigation = ITS_MITIGATION_OFF; +@@ -1296,6 +1305,18 @@ static void __init its_select_mitigation + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + set_return_thunk(its_return_thunk); + break; ++ case ITS_CMD_RSB_STUFF: ++ its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; ++ setup_force_cpu_cap(X86_FEATURE_RETHUNK); ++ setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); ++#ifdef CONFIG_CALL_DEPTH_TRACKING ++ set_return_thunk(&__x86_return_skl); ++#endif ++ if (retbleed_mitigation == RETBLEED_MITIGATION_NONE) { ++ retbleed_mitigation = RETBLEED_MITIGATION_STUFF; ++ pr_info("Retbleed mitigation updated to stuffing\n"); ++ } ++ break; + } + out: + pr_info("%s\n", its_strings[its_mitigation]); diff --git a/queue-6.6/x86-its-add-vmexit-option-to-skip-mitigation-on-some-cpus.patch b/queue-6.6/x86-its-add-vmexit-option-to-skip-mitigation-on-some-cpus.patch new file mode 100644 index 0000000000..4a53903377 --- /dev/null +++ b/queue-6.6/x86-its-add-vmexit-option-to-skip-mitigation-on-some-cpus.patch @@ -0,0 +1,140 @@ +From 027b33d1dd79f46d14d5fb8d2a7b13ea30a30100 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Mon, 18 Nov 2024 09:53:12 -0800 +Subject: x86/its: Add "vmexit" option to skip mitigation on some CPUs + +From: Pawan Gupta + +commit 2665281a07e19550944e8354a2024635a7b2714a upstream. + +Ice Lake generation CPUs are not affected by guest/host isolation part of +ITS. If a user is only concerned about KVM guests, they can now choose a +new cmdline option "vmexit" that will not deploy the ITS mitigation when +CPU is not affected by guest/host isolation. This saves the performance +overhead of ITS mitigation on Ice Lake gen CPUs. + +When "vmexit" option selected, if the CPU is affected by ITS guest/host +isolation, the default ITS mitigation is deployed. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/admin-guide/kernel-parameters.txt | 2 ++ + arch/x86/include/asm/cpufeatures.h | 1 + + arch/x86/kernel/cpu/bugs.c | 11 +++++++++++ + arch/x86/kernel/cpu/common.c | 19 ++++++++++++------- + 4 files changed, 26 insertions(+), 7 deletions(-) + +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -2068,6 +2068,8 @@ + off: Disable mitigation. + force: Force the ITS bug and deploy default + mitigation. ++ vmexit: Only deploy mitigation if CPU is affected by ++ guest/host isolation part of ITS. + + For details see: + Documentation/admin-guide/hw-vuln/indirect-target-selection.rst +--- a/arch/x86/include/asm/cpufeatures.h ++++ b/arch/x86/include/asm/cpufeatures.h +@@ -520,4 +520,5 @@ + #define X86_BUG_BHI X86_BUG(1*32 + 3) /* CPU is affected by Branch History Injection */ + #define X86_BUG_IBPB_NO_RET X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */ + #define X86_BUG_ITS X86_BUG(1*32 + 5) /* CPU is affected by Indirect Target Selection */ ++#define X86_BUG_ITS_NATIVE_ONLY X86_BUG(1*32 + 6) /* CPU is affected by ITS, VMX is not affected */ + #endif /* _ASM_X86_CPUFEATURES_H */ +--- a/arch/x86/kernel/cpu/bugs.c ++++ b/arch/x86/kernel/cpu/bugs.c +@@ -1188,16 +1188,19 @@ do_cmd_auto: + enum its_mitigation_cmd { + ITS_CMD_OFF, + ITS_CMD_ON, ++ ITS_CMD_VMEXIT, + }; + + enum its_mitigation { + ITS_MITIGATION_OFF, ++ ITS_MITIGATION_VMEXIT_ONLY, + ITS_MITIGATION_ALIGNED_THUNKS, + ITS_MITIGATION_RETPOLINE_STUFF, + }; + + static const char * const its_strings[] = { + [ITS_MITIGATION_OFF] = "Vulnerable", ++ [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", + [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", + [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", + }; +@@ -1224,6 +1227,8 @@ static int __init its_parse_cmdline(char + } else if (!strcmp(str, "force")) { + its_cmd = ITS_CMD_ON; + setup_force_cpu_bug(X86_BUG_ITS); ++ } else if (!strcmp(str, "vmexit")) { ++ its_cmd = ITS_CMD_VMEXIT; + } else { + pr_err("Ignoring unknown indirect_target_selection option (%s).", str); + } +@@ -1278,6 +1283,12 @@ static void __init its_select_mitigation + case ITS_CMD_OFF: + its_mitigation = ITS_MITIGATION_OFF; + break; ++ case ITS_CMD_VMEXIT: ++ if (boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) { ++ its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; ++ goto out; ++ } ++ fallthrough; + case ITS_CMD_ON: + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; + if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) +--- a/arch/x86/kernel/cpu/common.c ++++ b/arch/x86/kernel/cpu/common.c +@@ -1274,6 +1274,8 @@ static const __initconst struct x86_cpu_ + #define RFDS BIT(7) + /* CPU is affected by Indirect Target Selection */ + #define ITS BIT(8) ++/* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */ ++#define ITS_NATIVE_ONLY BIT(9) + + static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { + VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), +@@ -1294,16 +1296,16 @@ static const struct x86_cpu_id cpu_vuln_ + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0xc), MMIO | RETBLEED | GDS | SRBDS), + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | ITS), + VULNBL_INTEL_STEPPINGS(CANNONLAKE_L, X86_STEPPING_ANY, RETBLEED), +- VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), +- VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS | ITS), +- VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS | ITS | ITS_NATIVE_ONLY), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED | ITS), + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), +- VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS | ITS), +- VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS | ITS | ITS_NATIVE_ONLY), ++ VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED), +- VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(ALDERLAKE, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ALDERLAKE_L, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE, X86_STEPPING_ANY, RFDS), +@@ -1520,8 +1522,11 @@ static void __init cpu_set_bug_bits(stru + if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET)) + setup_force_cpu_bug(X86_BUG_IBPB_NO_RET); + +- if (vulnerable_to_its(x86_arch_cap_msr)) ++ if (vulnerable_to_its(x86_arch_cap_msr)) { + setup_force_cpu_bug(X86_BUG_ITS); ++ if (cpu_matches(cpu_vuln_blacklist, ITS_NATIVE_ONLY)) ++ setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY); ++ } + + if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) + return; diff --git a/queue-6.6/x86-its-align-rets-in-bhb-clear-sequence-to-avoid-thunking.patch b/queue-6.6/x86-its-align-rets-in-bhb-clear-sequence-to-avoid-thunking.patch new file mode 100644 index 0000000000..281a1e2b49 --- /dev/null +++ b/queue-6.6/x86-its-align-rets-in-bhb-clear-sequence-to-avoid-thunking.patch @@ -0,0 +1,100 @@ +From 4a8e573f1c5f9ef64275373ebde21bb4f05f74ba Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 2 May 2025 06:25:19 -0700 +Subject: x86/its: Align RETs in BHB clear sequence to avoid thunking + +From: Pawan Gupta + +commit f0cd7091cc5a032c8870b4285305d9172569d126 upstream. + +The software mitigation for BHI is to execute BHB clear sequence at syscall +entry, and possibly after a cBPF program. ITS mitigation thunks RETs in the +lower half of the cacheline. This causes the RETs in the BHB clear sequence +to be thunked as well, adding unnecessary branches to the BHB clear +sequence. + +Since the sequence is in hot path, align the RET instructions in the +sequence to avoid thunking. + +This is how disassembly clear_bhb_loop() looks like after this change: + + 0x44 <+4>: mov $0x5,%ecx + 0x49 <+9>: call 0xffffffff81001d9b + 0x4e <+14>: jmp 0xffffffff81001de5 + 0x53 <+19>: int3 + ... + 0x9b <+91>: call 0xffffffff81001dce + 0xa0 <+96>: ret + 0xa1 <+97>: int3 + ... + 0xce <+142>: mov $0x5,%eax + 0xd3 <+147>: jmp 0xffffffff81001dd6 + 0xd5 <+149>: nop + 0xd6 <+150>: sub $0x1,%eax + 0xd9 <+153>: jne 0xffffffff81001dd3 + 0xdb <+155>: sub $0x1,%ecx + 0xde <+158>: jne 0xffffffff81001d9b + 0xe0 <+160>: ret + 0xe1 <+161>: int3 + 0xe2 <+162>: int3 + 0xe3 <+163>: int3 + 0xe4 <+164>: int3 + 0xe5 <+165>: lfence + 0xe8 <+168>: pop %rbp + 0xe9 <+169>: ret + +Suggested-by: Andrew Cooper +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/entry/entry_64.S | 20 +++++++++++++++++--- + 1 file changed, 17 insertions(+), 3 deletions(-) + +--- a/arch/x86/entry/entry_64.S ++++ b/arch/x86/entry/entry_64.S +@@ -1569,7 +1569,9 @@ SYM_CODE_END(rewind_stack_and_make_dead) + * ORC to unwind properly. + * + * The alignment is for performance and not for safety, and may be safely +- * refactored in the future if needed. ++ * refactored in the future if needed. The .skips are for safety, to ensure ++ * that all RETs are in the second half of a cacheline to mitigate Indirect ++ * Target Selection, rather than taking the slowpath via its_return_thunk. + */ + SYM_FUNC_START(clear_bhb_loop) + push %rbp +@@ -1579,10 +1581,22 @@ SYM_FUNC_START(clear_bhb_loop) + call 1f + jmp 5f + .align 64, 0xcc ++ /* ++ * Shift instructions so that the RET is in the upper half of the ++ * cacheline and don't take the slowpath to its_return_thunk. ++ */ ++ .skip 32 - (.Lret1 - 1f), 0xcc + ANNOTATE_INTRA_FUNCTION_CALL + 1: call 2f +- RET ++.Lret1: RET + .align 64, 0xcc ++ /* ++ * As above shift instructions for RET at .Lret2 as well. ++ * ++ * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc ++ * but some Clang versions (e.g. 18) don't like this. ++ */ ++ .skip 32 - 18, 0xcc + 2: movl $5, %eax + 3: jmp 4f + nop +@@ -1590,7 +1604,7 @@ SYM_FUNC_START(clear_bhb_loop) + jnz 3b + sub $1, %ecx + jnz 1b +- RET ++.Lret2: RET + 5: lfence + pop %rbp + RET diff --git a/queue-6.6/x86-its-enable-indirect-target-selection-mitigation.patch b/queue-6.6/x86-its-enable-indirect-target-selection-mitigation.patch new file mode 100644 index 0000000000..ed2fbb7763 --- /dev/null +++ b/queue-6.6/x86-its-enable-indirect-target-selection-mitigation.patch @@ -0,0 +1,377 @@ +From 959e1efda0c5f9b896e103956e325620abfe7357 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 21 Jun 2024 20:23:23 -0700 +Subject: x86/its: Enable Indirect Target Selection mitigation + +From: Pawan Gupta + +commit f4818881c47fd91fcb6d62373c57c7844e3de1c0 upstream. + +Indirect Target Selection (ITS) is a bug in some pre-ADL Intel CPUs with +eIBRS. It affects prediction of indirect branch and RETs in the +lower half of cacheline. Due to ITS such branches may get wrongly predicted +to a target of (direct or indirect) branch that is located in the upper +half of the cacheline. + +Scope of impact +=============== + +Guest/host isolation +-------------------- +When eIBRS is used for guest/host isolation, the indirect branches in the +VMM may still be predicted with targets corresponding to branches in the +guest. + +Intra-mode +---------- +cBPF or other native gadgets can be used for intra-mode training and +disclosure using ITS. + +User/kernel isolation +--------------------- +When eIBRS is enabled user/kernel isolation is not impacted. + +Indirect Branch Prediction Barrier (IBPB) +----------------------------------------- +After an IBPB, indirect branches may be predicted with targets +corresponding to direct branches which were executed prior to IBPB. This is +mitigated by a microcode update. + +Add cmdline parameter indirect_target_selection=off|on|force to control the +mitigation to relocate the affected branches to an ITS-safe thunk i.e. +located in the upper half of cacheline. Also add the sysfs reporting. + +When retpoline mitigation is deployed, ITS safe-thunks are not needed, +because retpoline sequence is already ITS-safe. Similarly, when call depth +tracking (CDT) mitigation is deployed (retbleed=stuff), ITS safe return +thunk is not used, as CDT prevents RSB-underflow. + +To not overcomplicate things, ITS mitigation is not supported with +spectre-v2 lfence;jmp mitigation. Moreover, it is less practical to deploy +lfence;jmp mitigation on ITS affected parts anyways. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/ABI/testing/sysfs-devices-system-cpu | 1 + Documentation/admin-guide/kernel-parameters.txt | 13 + + arch/x86/include/asm/nospec-branch.h | 6 + arch/x86/kernel/cpu/bugs.c | 141 ++++++++++++++++++++- + drivers/base/cpu.c | 3 + include/linux/cpu.h | 2 + 6 files changed, 156 insertions(+), 10 deletions(-) + +--- a/Documentation/ABI/testing/sysfs-devices-system-cpu ++++ b/Documentation/ABI/testing/sysfs-devices-system-cpu +@@ -514,6 +514,7 @@ Description: information about CPUs hete + + What: /sys/devices/system/cpu/vulnerabilities + /sys/devices/system/cpu/vulnerabilities/gather_data_sampling ++ /sys/devices/system/cpu/vulnerabilities/indirect_target_selection + /sys/devices/system/cpu/vulnerabilities/itlb_multihit + /sys/devices/system/cpu/vulnerabilities/l1tf + /sys/devices/system/cpu/vulnerabilities/mds +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -2060,6 +2060,18 @@ + different crypto accelerators. This option can be used + to achieve best performance for particular HW. + ++ indirect_target_selection= [X86,Intel] Mitigation control for Indirect ++ Target Selection(ITS) bug in Intel CPUs. Updated ++ microcode is also required for a fix in IBPB. ++ ++ on: Enable mitigation (default). ++ off: Disable mitigation. ++ force: Force the ITS bug and deploy default ++ mitigation. ++ ++ For details see: ++ Documentation/admin-guide/hw-vuln/indirect-target-selection.rst ++ + init= [KNL] + Format: + Run specified binary instead of /sbin/init as init +@@ -3331,6 +3343,7 @@ + expose users to several CPU vulnerabilities. + Equivalent to: if nokaslr then kpti=0 [ARM64] + gather_data_sampling=off [X86] ++ indirect_target_selection=off [X86] + kvm.nx_huge_pages=off [X86] + l1tf=off [X86] + mds=off [X86] +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -421,11 +421,6 @@ extern void (*x86_return_thunk)(void); + #ifdef CONFIG_CALL_DEPTH_TRACKING + extern void __x86_return_skl(void); + +-static inline void x86_set_skl_return_thunk(void) +-{ +- x86_return_thunk = &__x86_return_skl; +-} +- + #define CALL_DEPTH_ACCOUNT \ + ALTERNATIVE("", \ + __stringify(INCREMENT_CALL_DEPTH), \ +@@ -438,7 +433,6 @@ DECLARE_PER_CPU(u64, __x86_stuffs_count) + DECLARE_PER_CPU(u64, __x86_ctxsw_count); + #endif + #else +-static inline void x86_set_skl_return_thunk(void) {} + + #define CALL_DEPTH_ACCOUNT "" + +--- a/arch/x86/kernel/cpu/bugs.c ++++ b/arch/x86/kernel/cpu/bugs.c +@@ -49,6 +49,7 @@ static void __init srbds_select_mitigati + static void __init l1d_flush_select_mitigation(void); + static void __init srso_select_mitigation(void); + static void __init gds_select_mitigation(void); ++static void __init its_select_mitigation(void); + + /* The base value of the SPEC_CTRL MSR without task-specific bits set */ + u64 x86_spec_ctrl_base; +@@ -67,6 +68,14 @@ static DEFINE_MUTEX(spec_ctrl_mutex); + + void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; + ++static void __init set_return_thunk(void *thunk) ++{ ++ if (x86_return_thunk != __x86_return_thunk) ++ pr_warn("x86/bugs: return thunk changed\n"); ++ ++ x86_return_thunk = thunk; ++} ++ + /* Update SPEC_CTRL MSR and its cached copy unconditionally */ + static void update_spec_ctrl(u64 val) + { +@@ -175,6 +184,7 @@ void __init cpu_select_mitigations(void) + */ + srso_select_mitigation(); + gds_select_mitigation(); ++ its_select_mitigation(); + } + + /* +@@ -1102,7 +1112,7 @@ do_cmd_auto: + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_UNRET); + +- x86_return_thunk = retbleed_return_thunk; ++ set_return_thunk(retbleed_return_thunk); + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && + boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) +@@ -1136,7 +1146,9 @@ do_cmd_auto: + case RETBLEED_MITIGATION_STUFF: + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); +- x86_set_skl_return_thunk(); ++#ifdef CONFIG_CALL_DEPTH_TRACKING ++ set_return_thunk(&__x86_return_skl); ++#endif + break; + + default: +@@ -1171,6 +1183,114 @@ do_cmd_auto: + } + + #undef pr_fmt ++#define pr_fmt(fmt) "ITS: " fmt ++ ++enum its_mitigation_cmd { ++ ITS_CMD_OFF, ++ ITS_CMD_ON, ++}; ++ ++enum its_mitigation { ++ ITS_MITIGATION_OFF, ++ ITS_MITIGATION_ALIGNED_THUNKS, ++ ITS_MITIGATION_RETPOLINE_STUFF, ++}; ++ ++static const char * const its_strings[] = { ++ [ITS_MITIGATION_OFF] = "Vulnerable", ++ [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", ++ [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", ++}; ++ ++static enum its_mitigation its_mitigation __ro_after_init = ITS_MITIGATION_ALIGNED_THUNKS; ++ ++static enum its_mitigation_cmd its_cmd __ro_after_init = ++ IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_CMD_ON : ITS_CMD_OFF; ++ ++static int __init its_parse_cmdline(char *str) ++{ ++ if (!str) ++ return -EINVAL; ++ ++ if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { ++ pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); ++ return 0; ++ } ++ ++ if (!strcmp(str, "off")) { ++ its_cmd = ITS_CMD_OFF; ++ } else if (!strcmp(str, "on")) { ++ its_cmd = ITS_CMD_ON; ++ } else if (!strcmp(str, "force")) { ++ its_cmd = ITS_CMD_ON; ++ setup_force_cpu_bug(X86_BUG_ITS); ++ } else { ++ pr_err("Ignoring unknown indirect_target_selection option (%s).", str); ++ } ++ ++ return 0; ++} ++early_param("indirect_target_selection", its_parse_cmdline); ++ ++static void __init its_select_mitigation(void) ++{ ++ enum its_mitigation_cmd cmd = its_cmd; ++ ++ if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) { ++ its_mitigation = ITS_MITIGATION_OFF; ++ return; ++ } ++ ++ /* Retpoline+CDT mitigates ITS, bail out */ ++ if (boot_cpu_has(X86_FEATURE_RETPOLINE) && ++ boot_cpu_has(X86_FEATURE_CALL_DEPTH)) { ++ its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; ++ goto out; ++ } ++ ++ /* Exit early to avoid irrelevant warnings */ ++ if (cmd == ITS_CMD_OFF) { ++ its_mitigation = ITS_MITIGATION_OFF; ++ goto out; ++ } ++ if (spectre_v2_enabled == SPECTRE_V2_NONE) { ++ pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); ++ its_mitigation = ITS_MITIGATION_OFF; ++ goto out; ++ } ++ if (!IS_ENABLED(CONFIG_RETPOLINE) || !IS_ENABLED(CONFIG_RETHUNK)) { ++ pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); ++ its_mitigation = ITS_MITIGATION_OFF; ++ goto out; ++ } ++ if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { ++ pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); ++ its_mitigation = ITS_MITIGATION_OFF; ++ goto out; ++ } ++ if (boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { ++ pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); ++ its_mitigation = ITS_MITIGATION_OFF; ++ goto out; ++ } ++ ++ switch (cmd) { ++ case ITS_CMD_OFF: ++ its_mitigation = ITS_MITIGATION_OFF; ++ break; ++ case ITS_CMD_ON: ++ its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; ++ if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) ++ setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); ++ setup_force_cpu_cap(X86_FEATURE_RETHUNK); ++ set_return_thunk(its_return_thunk); ++ break; ++ } ++out: ++ pr_info("%s\n", its_strings[its_mitigation]); ++} ++ ++#undef pr_fmt + #define pr_fmt(fmt) "Spectre V2 : " fmt + + static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = +@@ -2607,10 +2727,10 @@ static void __init srso_select_mitigatio + + if (boot_cpu_data.x86 == 0x19) { + setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); +- x86_return_thunk = srso_alias_return_thunk; ++ set_return_thunk(srso_alias_return_thunk); + } else { + setup_force_cpu_cap(X86_FEATURE_SRSO); +- x86_return_thunk = srso_return_thunk; ++ set_return_thunk(srso_return_thunk); + } + if (has_microcode) + srso_mitigation = SRSO_MITIGATION_SAFE_RET; +@@ -2794,6 +2914,11 @@ static ssize_t rfds_show_state(char *buf + return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); + } + ++static ssize_t its_show_state(char *buf) ++{ ++ return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); ++} ++ + static char *stibp_state(void) + { + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && +@@ -2976,6 +3101,9 @@ static ssize_t cpu_show_common(struct de + case X86_BUG_RFDS: + return rfds_show_state(buf); + ++ case X86_BUG_ITS: ++ return its_show_state(buf); ++ + default: + break; + } +@@ -3055,4 +3183,9 @@ ssize_t cpu_show_reg_file_data_sampling( + { + return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); + } ++ ++ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ return cpu_show_common(dev, attr, buf, X86_BUG_ITS); ++} + #endif +--- a/drivers/base/cpu.c ++++ b/drivers/base/cpu.c +@@ -566,6 +566,7 @@ CPU_SHOW_VULN_FALLBACK(retbleed); + CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow); + CPU_SHOW_VULN_FALLBACK(gds); + CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling); ++CPU_SHOW_VULN_FALLBACK(indirect_target_selection); + + static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); + static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); +@@ -581,6 +582,7 @@ static DEVICE_ATTR(retbleed, 0444, cpu_s + static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL); + static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL); + static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL); ++static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL); + + static struct attribute *cpu_root_vulnerabilities_attrs[] = { + &dev_attr_meltdown.attr, +@@ -597,6 +599,7 @@ static struct attribute *cpu_root_vulner + &dev_attr_spec_rstack_overflow.attr, + &dev_attr_gather_data_sampling.attr, + &dev_attr_reg_file_data_sampling.attr, ++ &dev_attr_indirect_target_selection.attr, + NULL + }; + +--- a/include/linux/cpu.h ++++ b/include/linux/cpu.h +@@ -77,6 +77,8 @@ extern ssize_t cpu_show_gds(struct devic + struct device_attribute *attr, char *buf); + extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev, + struct device_attribute *attr, char *buf); ++extern ssize_t cpu_show_indirect_target_selection(struct device *dev, ++ struct device_attribute *attr, char *buf); + + extern __printf(4, 5) + struct device *cpu_device_create(struct device *parent, void *drvdata, diff --git a/queue-6.6/x86-its-enumerate-indirect-target-selection-its-bug.patch b/queue-6.6/x86-its-enumerate-indirect-target-selection-its-bug.patch new file mode 100644 index 0000000000..120c6986a9 --- /dev/null +++ b/queue-6.6/x86-its-enumerate-indirect-target-selection-its-bug.patch @@ -0,0 +1,164 @@ +From 1efca446733aebce4322bb733978cccc521054e3 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 21 Jun 2024 17:40:41 -0700 +Subject: x86/its: Enumerate Indirect Target Selection (ITS) bug + +From: Pawan Gupta + +commit 159013a7ca18c271ff64192deb62a689b622d860 upstream. + +ITS bug in some pre-Alderlake Intel CPUs may allow indirect branches in the +first half of a cache line get predicted to a target of a branch located in +the second half of the cache line. + +Set X86_BUG_ITS on affected CPUs. Mitigation to follow in later commits. + +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Josh Poimboeuf +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/msr-index.h | 8 +++++ + arch/x86/kernel/cpu/common.c | 58 +++++++++++++++++++++++++++++-------- + arch/x86/kvm/x86.c | 4 +- + 4 files changed, 58 insertions(+), 13 deletions(-) + +--- a/arch/x86/include/asm/cpufeatures.h ++++ b/arch/x86/include/asm/cpufeatures.h +@@ -518,4 +518,5 @@ + #define X86_BUG_RFDS X86_BUG(1*32 + 2) /* CPU is vulnerable to Register File Data Sampling */ + #define X86_BUG_BHI X86_BUG(1*32 + 3) /* CPU is affected by Branch History Injection */ + #define X86_BUG_IBPB_NO_RET X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */ ++#define X86_BUG_ITS X86_BUG(1*32 + 5) /* CPU is affected by Indirect Target Selection */ + #endif /* _ASM_X86_CPUFEATURES_H */ +--- a/arch/x86/include/asm/msr-index.h ++++ b/arch/x86/include/asm/msr-index.h +@@ -180,6 +180,14 @@ + * VERW clears CPU Register + * File. + */ ++#define ARCH_CAP_ITS_NO BIT_ULL(62) /* ++ * Not susceptible to ++ * Indirect Target Selection. ++ * This bit is not set by ++ * HW, but is synthesized by ++ * VMMs for guests to know ++ * their affected status. ++ */ + + #define ARCH_CAP_XAPIC_DISABLE BIT(21) /* + * IA32_XAPIC_DISABLE_STATUS MSR +--- a/arch/x86/kernel/cpu/common.c ++++ b/arch/x86/kernel/cpu/common.c +@@ -1272,6 +1272,8 @@ static const __initconst struct x86_cpu_ + #define GDS BIT(6) + /* CPU is affected by Register File Data Sampling */ + #define RFDS BIT(7) ++/* CPU is affected by Indirect Target Selection */ ++#define ITS BIT(8) + + static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { + VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), +@@ -1283,22 +1285,25 @@ static const struct x86_cpu_id cpu_vuln_ + VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS), + VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO), + VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS), +- VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED | GDS), ++ VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x0, 0x5), MMIO | RETBLEED | GDS), ++ VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS), + VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), + VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), +- VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), +- VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), ++ VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x0, 0xb), MMIO | RETBLEED | GDS | SRBDS), ++ VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | ITS), ++ VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0xc), MMIO | RETBLEED | GDS | SRBDS), ++ VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | ITS), + VULNBL_INTEL_STEPPINGS(CANNONLAKE_L, X86_STEPPING_ANY, RETBLEED), +- VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), +- VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS), +- VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS), +- VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), +- VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED), +- VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), +- VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS), +- VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED | ITS), ++ VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS | ITS), ++ VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS | ITS), + VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED), +- VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS), ++ VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS), + VULNBL_INTEL_STEPPINGS(ALDERLAKE, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ALDERLAKE_L, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE, X86_STEPPING_ANY, RFDS), +@@ -1362,6 +1367,32 @@ static bool __init vulnerable_to_rfds(u6 + return cpu_matches(cpu_vuln_blacklist, RFDS); + } + ++static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) ++{ ++ /* The "immunity" bit trumps everything else: */ ++ if (x86_arch_cap_msr & ARCH_CAP_ITS_NO) ++ return false; ++ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) ++ return false; ++ ++ /* None of the affected CPUs have BHI_CTRL */ ++ if (boot_cpu_has(X86_FEATURE_BHI_CTRL)) ++ return false; ++ ++ /* ++ * If a VMM did not expose ITS_NO, assume that a guest could ++ * be running on a vulnerable hardware or may migrate to such ++ * hardware. ++ */ ++ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) ++ return true; ++ ++ if (cpu_matches(cpu_vuln_blacklist, ITS)) ++ return true; ++ ++ return false; ++} ++ + static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) + { + u64 x86_arch_cap_msr = x86_read_arch_cap_msr(); +@@ -1489,6 +1520,9 @@ static void __init cpu_set_bug_bits(stru + if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET)) + setup_force_cpu_bug(X86_BUG_IBPB_NO_RET); + ++ if (vulnerable_to_its(x86_arch_cap_msr)) ++ setup_force_cpu_bug(X86_BUG_ITS); ++ + if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) + return; + +--- a/arch/x86/kvm/x86.c ++++ b/arch/x86/kvm/x86.c +@@ -1621,7 +1621,7 @@ static bool kvm_is_immutable_feature_msr + ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ + ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ + ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \ +- ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO) ++ ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO) + + static u64 kvm_get_arch_capabilities(void) + { +@@ -1655,6 +1655,8 @@ static u64 kvm_get_arch_capabilities(voi + data |= ARCH_CAP_MDS_NO; + if (!boot_cpu_has_bug(X86_BUG_RFDS)) + data |= ARCH_CAP_RFDS_NO; ++ if (!boot_cpu_has_bug(X86_BUG_ITS)) ++ data |= ARCH_CAP_ITS_NO; + + if (!boot_cpu_has(X86_FEATURE_RTM)) { + /* diff --git a/queue-6.6/x86-its-use-dynamic-thunks-for-indirect-branches.patch b/queue-6.6/x86-its-use-dynamic-thunks-for-indirect-branches.patch new file mode 100644 index 0000000000..240d0d02ac --- /dev/null +++ b/queue-6.6/x86-its-use-dynamic-thunks-for-indirect-branches.patch @@ -0,0 +1,265 @@ +From 93d8493b1f427133cc9074f5c136aee1dc256ab3 Mon Sep 17 00:00:00 2001 +From: Peter Zijlstra +Date: Mon, 14 Oct 2024 10:05:48 -0700 +Subject: x86/its: Use dynamic thunks for indirect branches + +From: Peter Zijlstra + +commit 872df34d7c51a79523820ea6a14860398c639b87 upstream. + +ITS mitigation moves the unsafe indirect branches to a safe thunk. This +could degrade the prediction accuracy as the source address of indirect +branches becomes same for different execution paths. + +To improve the predictions, and hence the performance, assign a separate +thunk for each indirect callsite. This is also a defense-in-depth measure +to avoid indirect branches aliasing with each other. + +As an example, 5000 dynamic thunks would utilize around 16 bits of the +address space, thereby gaining entropy. For a BTB that uses +32 bits for indexing, dynamic thunks could provide better prediction +accuracy over fixed thunks. + +Have ITS thunks be variable sized and use EXECMEM_MODULE_TEXT such that +they are both more flexible (got to extend them later) and live in 2M TLBs, +just like kernel code, avoiding undue TLB pressure. + + [ pawan: CONFIG_EXECMEM and CONFIG_EXECMEM_ROX are not supported on + backport kernel, made changes to use module_alloc() and + set_memory_*() for dynamic thunks. ] + +Signed-off-by: Peter Zijlstra (Intel) +Signed-off-by: Pawan Gupta +Signed-off-by: Dave Hansen +Reviewed-by: Alexandre Chartre +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/alternative.h | 10 ++ + arch/x86/kernel/alternative.c | 132 ++++++++++++++++++++++++++++++++++++- + arch/x86/kernel/module.c | 7 + + include/linux/module.h | 5 + + 4 files changed, 151 insertions(+), 3 deletions(-) + +--- a/arch/x86/include/asm/alternative.h ++++ b/arch/x86/include/asm/alternative.h +@@ -130,6 +130,16 @@ static __always_inline int x86_call_dept + } + #endif + ++#ifdef CONFIG_MITIGATION_ITS ++extern void its_init_mod(struct module *mod); ++extern void its_fini_mod(struct module *mod); ++extern void its_free_mod(struct module *mod); ++#else /* CONFIG_MITIGATION_ITS */ ++static inline void its_init_mod(struct module *mod) { } ++static inline void its_fini_mod(struct module *mod) { } ++static inline void its_free_mod(struct module *mod) { } ++#endif ++ + #if defined(CONFIG_RETHUNK) && defined(CONFIG_OBJTOOL) + extern bool cpu_wants_rethunk(void); + extern bool cpu_wants_rethunk_at(void *addr); +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -18,6 +18,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -31,6 +33,7 @@ + #include + #include + #include ++#include + + int __read_mostly alternatives_patched; + +@@ -124,6 +127,125 @@ const unsigned char * const x86_nops[ASM + #endif + }; + ++#ifdef CONFIG_MITIGATION_ITS ++ ++static struct module *its_mod; ++static void *its_page; ++static unsigned int its_offset; ++ ++/* Initialize a thunk with the "jmp *reg; int3" instructions. */ ++static void *its_init_thunk(void *thunk, int reg) ++{ ++ u8 *bytes = thunk; ++ int i = 0; ++ ++ if (reg >= 8) { ++ bytes[i++] = 0x41; /* REX.B prefix */ ++ reg -= 8; ++ } ++ bytes[i++] = 0xff; ++ bytes[i++] = 0xe0 + reg; /* jmp *reg */ ++ bytes[i++] = 0xcc; ++ ++ return thunk; ++} ++ ++void its_init_mod(struct module *mod) ++{ ++ if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) ++ return; ++ ++ mutex_lock(&text_mutex); ++ its_mod = mod; ++ its_page = NULL; ++} ++ ++void its_fini_mod(struct module *mod) ++{ ++ if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) ++ return; ++ ++ WARN_ON_ONCE(its_mod != mod); ++ ++ its_mod = NULL; ++ its_page = NULL; ++ mutex_unlock(&text_mutex); ++ ++ for (int i = 0; i < mod->its_num_pages; i++) { ++ void *page = mod->its_page_array[i]; ++ set_memory_rox((unsigned long)page, 1); ++ } ++} ++ ++void its_free_mod(struct module *mod) ++{ ++ if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) ++ return; ++ ++ for (int i = 0; i < mod->its_num_pages; i++) { ++ void *page = mod->its_page_array[i]; ++ module_memfree(page); ++ } ++ kfree(mod->its_page_array); ++} ++ ++DEFINE_FREE(its_execmem, void *, if (_T) module_memfree(_T)); ++ ++static void *its_alloc(void) ++{ ++ void *page __free(its_execmem) = module_alloc(PAGE_SIZE); ++ ++ if (!page) ++ return NULL; ++ ++ if (its_mod) { ++ void *tmp = krealloc(its_mod->its_page_array, ++ (its_mod->its_num_pages+1) * sizeof(void *), ++ GFP_KERNEL); ++ if (!tmp) ++ return NULL; ++ ++ its_mod->its_page_array = tmp; ++ its_mod->its_page_array[its_mod->its_num_pages++] = page; ++ } ++ ++ return no_free_ptr(page); ++} ++ ++static void *its_allocate_thunk(int reg) ++{ ++ int size = 3 + (reg / 8); ++ void *thunk; ++ ++ if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) { ++ its_page = its_alloc(); ++ if (!its_page) { ++ pr_err("ITS page allocation failed\n"); ++ return NULL; ++ } ++ memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE); ++ its_offset = 32; ++ } ++ ++ /* ++ * If the indirect branch instruction will be in the lower half ++ * of a cacheline, then update the offset to reach the upper half. ++ */ ++ if ((its_offset + size - 1) % 64 < 32) ++ its_offset = ((its_offset - 1) | 0x3F) + 33; ++ ++ thunk = its_page + its_offset; ++ its_offset += size; ++ ++ set_memory_rw((unsigned long)its_page, 1); ++ thunk = its_init_thunk(thunk, reg); ++ set_memory_rox((unsigned long)its_page, 1); ++ ++ return thunk; ++} ++ ++#endif ++ + /* + * Fill the buffer with a single effective instruction of size @len. + * +@@ -577,9 +699,13 @@ static int emit_call_track_retpoline(voi + #ifdef CONFIG_MITIGATION_ITS + static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) + { +- return __emit_trampoline(addr, insn, bytes, +- __x86_indirect_its_thunk_array[reg], +- __x86_indirect_its_thunk_array[reg]); ++ u8 *thunk = __x86_indirect_its_thunk_array[reg]; ++ u8 *tmp = its_allocate_thunk(reg); ++ ++ if (tmp) ++ thunk = tmp; ++ ++ return __emit_trampoline(addr, insn, bytes, thunk, thunk); + } + + /* Check if an indirect branch is at ITS-unsafe address */ +--- a/arch/x86/kernel/module.c ++++ b/arch/x86/kernel/module.c +@@ -312,6 +312,9 @@ int module_finalize(const Elf_Ehdr *hdr, + void *pseg = (void *)para->sh_addr; + apply_paravirt(pseg, pseg + para->sh_size); + } ++ ++ its_init_mod(me); ++ + if (retpolines || cfi) { + void *rseg = NULL, *cseg = NULL; + unsigned int rsize = 0, csize = 0; +@@ -332,6 +335,9 @@ int module_finalize(const Elf_Ehdr *hdr, + void *rseg = (void *)retpolines->sh_addr; + apply_retpolines(rseg, rseg + retpolines->sh_size); + } ++ ++ its_fini_mod(me); ++ + if (returns) { + void *rseg = (void *)returns->sh_addr; + apply_returns(rseg, rseg + returns->sh_size); +@@ -379,4 +385,5 @@ int module_finalize(const Elf_Ehdr *hdr, + void module_arch_cleanup(struct module *mod) + { + alternatives_smp_module_del(mod); ++ its_free_mod(mod); + } +--- a/include/linux/module.h ++++ b/include/linux/module.h +@@ -572,6 +572,11 @@ struct module { + atomic_t refcnt; + #endif + ++#ifdef CONFIG_MITIGATION_ITS ++ int its_num_pages; ++ void **its_page_array; ++#endif ++ + #ifdef CONFIG_CONSTRUCTORS + /* Constructor functions. */ + ctor_fn_t *ctors; diff --git a/queue-6.6/x86-speculation-add-a-conditional-cs-prefix-to-call_nospec.patch b/queue-6.6/x86-speculation-add-a-conditional-cs-prefix-to-call_nospec.patch new file mode 100644 index 0000000000..8ad70aa332 --- /dev/null +++ b/queue-6.6/x86-speculation-add-a-conditional-cs-prefix-to-call_nospec.patch @@ -0,0 +1,69 @@ +From 6de646645a97edbdd24de4fbf87dbf0564774a21 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 28 Feb 2025 18:35:58 -0800 +Subject: x86/speculation: Add a conditional CS prefix to CALL_NOSPEC + +From: Pawan Gupta + +commit 052040e34c08428a5a388b85787e8531970c0c67 upstream. + +Retpoline mitigation for spectre-v2 uses thunks for indirect branches. To +support this mitigation compilers add a CS prefix with +-mindirect-branch-cs-prefix. For an indirect branch in asm, this needs to +be added manually. + +CS prefix is already being added to indirect branches in asm files, but not +in inline asm. Add CS prefix to CALL_NOSPEC for inline asm as well. There +is no JMP_NOSPEC for inline asm. + +Reported-by: Josh Poimboeuf +Signed-off-by: Pawan Gupta +Signed-off-by: Ingo Molnar +Cc: Andrew Cooper +Cc: Peter Zijlstra +Link: https://lore.kernel.org/r/20250228-call-nospec-v3-2-96599fed0f33@linux.intel.com +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/nospec-branch.h | 19 +++++++++++++++---- + 1 file changed, 15 insertions(+), 4 deletions(-) + +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -219,9 +219,8 @@ + .endm + + /* +- * Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call +- * to the retpoline thunk with a CS prefix when the register requires +- * a RAX prefix byte to encode. Also see apply_retpolines(). ++ * Emits a conditional CS prefix that is compatible with ++ * -mindirect-branch-cs-prefix. + */ + .macro __CS_PREFIX reg:req + .irp rs,r8,r9,r10,r11,r12,r13,r14,r15 +@@ -455,11 +454,23 @@ static inline void x86_set_skl_return_th + #ifdef CONFIG_X86_64 + + /* ++ * Emits a conditional CS prefix that is compatible with ++ * -mindirect-branch-cs-prefix. ++ */ ++#define __CS_PREFIX(reg) \ ++ ".irp rs,r8,r9,r10,r11,r12,r13,r14,r15\n" \ ++ ".ifc \\rs," reg "\n" \ ++ ".byte 0x2e\n" \ ++ ".endif\n" \ ++ ".endr\n" ++ ++/* + * Inline asm uses the %V modifier which is only in newer GCC + * which is ensured when CONFIG_RETPOLINE is defined. + */ + #ifdef CONFIG_RETPOLINE +-#define CALL_NOSPEC "call __x86_indirect_thunk_%V[thunk_target]\n" ++#define CALL_NOSPEC __CS_PREFIX("%V[thunk_target]") \ ++ "call __x86_indirect_thunk_%V[thunk_target]\n" + #else + #define CALL_NOSPEC "call *%[thunk_target]\n" + #endif diff --git a/queue-6.6/x86-speculation-remove-the-extra-ifdef-around-call_nospec.patch b/queue-6.6/x86-speculation-remove-the-extra-ifdef-around-call_nospec.patch new file mode 100644 index 0000000000..16f4af23b1 --- /dev/null +++ b/queue-6.6/x86-speculation-remove-the-extra-ifdef-around-call_nospec.patch @@ -0,0 +1,56 @@ +From 4594194ff291eced4f7e0d817a1f5b6deb3c892a Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Thu, 20 Mar 2025 11:13:15 -0700 +Subject: x86/speculation: Remove the extra #ifdef around CALL_NOSPEC + +From: Pawan Gupta + +commit c8c81458863ab686cda4fe1e603fccaae0f12460 upstream. + +Commit: + + 010c4a461c1d ("x86/speculation: Simplify and make CALL_NOSPEC consistent") + +added an #ifdef CONFIG_RETPOLINE around the CALL_NOSPEC definition. This is +not required as this code is already under a larger #ifdef. + +Remove the extra #ifdef, no functional change. + +vmlinux size remains same before and after this change: + + CONFIG_RETPOLINE=y: + text data bss dec hex filename + 25434752 7342290 2301212 35078254 217406e vmlinux.before + 25434752 7342290 2301212 35078254 217406e vmlinux.after + + # CONFIG_RETPOLINE is not set: + text data bss dec hex filename + 22943094 6214994 1550152 30708240 1d49210 vmlinux.before + 22943094 6214994 1550152 30708240 1d49210 vmlinux.after + + [ pawan: s/CONFIG_MITIGATION_RETPOLINE/CONFIG_RETPOLINE/ ] + +Signed-off-by: Pawan Gupta +Signed-off-by: Ingo Molnar +Reviewed-by: Josh Poimboeuf +Link: https://lore.kernel.org/r/20250320-call-nospec-extra-ifdef-v1-1-d9b084d24820@linux.intel.com +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/nospec-branch.h | 4 ---- + 1 file changed, 4 deletions(-) + +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -468,12 +468,8 @@ static inline void x86_set_skl_return_th + * Inline asm uses the %V modifier which is only in newer GCC + * which is ensured when CONFIG_RETPOLINE is defined. + */ +-#ifdef CONFIG_RETPOLINE + #define CALL_NOSPEC __CS_PREFIX("%V[thunk_target]") \ + "call __x86_indirect_thunk_%V[thunk_target]\n" +-#else +-#define CALL_NOSPEC "call *%[thunk_target]\n" +-#endif + + # define THUNK_TARGET(addr) [thunk_target] "r" (addr) + diff --git a/queue-6.6/x86-speculation-simplify-and-make-call_nospec-consistent.patch b/queue-6.6/x86-speculation-simplify-and-make-call_nospec-consistent.patch new file mode 100644 index 0000000000..3d7150eece --- /dev/null +++ b/queue-6.6/x86-speculation-simplify-and-make-call_nospec-consistent.patch @@ -0,0 +1,58 @@ +From 447fba09db1fcd6f1dc90e3fa691be200ebe19a5 Mon Sep 17 00:00:00 2001 +From: Pawan Gupta +Date: Fri, 28 Feb 2025 18:35:43 -0800 +Subject: x86/speculation: Simplify and make CALL_NOSPEC consistent + +From: Pawan Gupta + +commit cfceff8526a426948b53445c02bcb98453c7330d upstream. + +CALL_NOSPEC macro is used to generate Spectre-v2 mitigation friendly +indirect branches. At compile time the macro defaults to indirect branch, +and at runtime those can be patched to thunk based mitigations. + +This approach is opposite of what is done for the rest of the kernel, where +the compile time default is to replace indirect calls with retpoline thunk +calls. + +Make CALL_NOSPEC consistent with the rest of the kernel, default to +retpoline thunk at compile time when CONFIG_RETPOLINE is +enabled. + + [ pawan: s/CONFIG_MITIGATION_RETPOLINE/CONFIG_RETPOLINE/ ] + +Signed-off-by: Pawan Gupta +Signed-off-by: Ingo Molnar +Cc: Andrew Cooper +Cc: Peter Zijlstra +Link: https://lore.kernel.org/r/20250228-call-nospec-v3-1-96599fed0f33@linux.intel.com +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/nospec-branch.h | 15 +++++---------- + 1 file changed, 5 insertions(+), 10 deletions(-) + +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -458,16 +458,11 @@ static inline void x86_set_skl_return_th + * Inline asm uses the %V modifier which is only in newer GCC + * which is ensured when CONFIG_RETPOLINE is defined. + */ +-# define CALL_NOSPEC \ +- ALTERNATIVE_2( \ +- ANNOTATE_RETPOLINE_SAFE \ +- "call *%[thunk_target]\n", \ +- "call __x86_indirect_thunk_%V[thunk_target]\n", \ +- X86_FEATURE_RETPOLINE, \ +- "lfence;\n" \ +- ANNOTATE_RETPOLINE_SAFE \ +- "call *%[thunk_target]\n", \ +- X86_FEATURE_RETPOLINE_LFENCE) ++#ifdef CONFIG_RETPOLINE ++#define CALL_NOSPEC "call __x86_indirect_thunk_%V[thunk_target]\n" ++#else ++#define CALL_NOSPEC "call *%[thunk_target]\n" ++#endif + + # define THUNK_TARGET(addr) [thunk_target] "r" (addr) +