releases/6.6.26/x86-bhi-add-bhi-mitigation-knob.patch

   1 From 40b7edcccbcf17c3bbe00341a042222933dba652 Mon Sep 17 00:00:00 2001
   2 From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
   3 Date: Mon, 11 Mar 2024 08:57:05 -0700
   4 Subject: x86/bhi: Add BHI mitigation knob
   5
   6 From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
   7
   8 commit ec9404e40e8f36421a2b66ecb76dc2209fe7f3ef upstream.
   9
  10 Branch history clearing software sequences and hardware control
  11 BHI_DIS_S were defined to mitigate Branch History Injection (BHI).
  12
  13 Add cmdline spectre_bhi={on|off|auto} to control BHI mitigation:
  14
  15  auto - Deploy the hardware mitigation BHI_DIS_S, if available.
  16  on   - Deploy the hardware mitigation BHI_DIS_S, if available,
  17         otherwise deploy the software sequence at syscall entry and
  18         VMexit.
  19  off  - Turn off BHI mitigation.
  20
  21 The default is auto mode which does not deploy the software sequence
  22 mitigation.  This is because of the hardening done in the syscall
  23 dispatch path, which is the likely target of BHI.
  24
  25 Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
  26 Signed-off-by: Daniel Sneddon <daniel.sneddon@linux.intel.com>
  27 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  28 Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
  29 Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
  30 Signed-off-by: Daniel Sneddon <daniel.sneddon@linux.intel.com>
  31 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  32 ---
  33  Documentation/admin-guide/hw-vuln/spectre.rst   |   45 ++++++++++--
  34  Documentation/admin-guide/kernel-parameters.txt |   11 ++
  35  arch/x86/Kconfig                                |   25 ++++++
  36  arch/x86/include/asm/cpufeatures.h              |    1
  37  arch/x86/kernel/cpu/bugs.c                      |   90 +++++++++++++++++++++++-
  38  5 files changed, 165 insertions(+), 7 deletions(-)
  39
  40 --- a/Documentation/admin-guide/hw-vuln/spectre.rst
  41 +++ b/Documentation/admin-guide/hw-vuln/spectre.rst
  42 @@ -138,11 +138,10 @@ associated with the source address of th
  43  the BHB might be shared across privilege levels even in the presence of
  44  Enhanced IBRS.
  45
  46 -Currently the only known real-world BHB attack vector is via
  47 -unprivileged eBPF. Therefore, it's highly recommended to not enable
  48 -unprivileged eBPF, especially when eIBRS is used (without retpolines).
  49 -For a full mitigation against BHB attacks, it's recommended to use
  50 -retpolines (or eIBRS combined with retpolines).
  51 +Previously the only known real-world BHB attack vector was via unprivileged
  52 +eBPF. Further research has found attacks that don't require unprivileged eBPF.
  53 +For a full mitigation against BHB attacks it is recommended to set BHI_DIS_S or
  54 +use the BHB clearing sequence.
  55
  56  Attack scenarios
  57  ----------------
  58 @@ -430,6 +429,21 @@ The possible values in this file are:
  59    'PBRSB-eIBRS: Not affected'  CPU is not affected by PBRSB
  60    ===========================  =======================================================
  61
  62 +  - Branch History Injection (BHI) protection status:
  63 +
  64 +.. list-table::
  65 +
  66 + * - BHI: Not affected
  67 +   - System is not affected
  68 + * - BHI: Retpoline
  69 +   - System is protected by retpoline
  70 + * - BHI: BHI_DIS_S
  71 +   - System is protected by BHI_DIS_S
  72 + * - BHI: SW loop
  73 +   - System is protected by software clearing sequence
  74 + * - BHI: Syscall hardening
  75 +   - Syscalls are hardened against BHI
  76 +
  77  Full mitigation might require a microcode update from the CPU
  78  vendor. When the necessary microcode is not available, the kernel will
  79  report vulnerability.
  80 @@ -484,7 +498,11 @@ Spectre variant 2
  81
  82     Systems which support enhanced IBRS (eIBRS) enable IBRS protection once at
  83     boot, by setting the IBRS bit, and they're automatically protected against
  84 -   Spectre v2 variant attacks.
  85 +   some Spectre v2 variant attacks. The BHB can still influence the choice of
  86 +   indirect branch predictor entry, and although branch predictor entries are
  87 +   isolated between modes when eIBRS is enabled, the BHB itself is not isolated
  88 +   between modes. Systems which support BHI_DIS_S will set it to protect against
  89 +   BHI attacks.
  90
  91     On Intel's enhanced IBRS systems, this includes cross-thread branch target
  92     injections on SMT systems (STIBP). In other words, Intel eIBRS enables
  93 @@ -638,6 +656,21 @@ kernel command line.
  94                 spectre_v2=off. Spectre variant 1 mitigations
  95                 cannot be disabled.
  96
  97 +       spectre_bhi=
  98 +
  99 +               [X86] Control mitigation of Branch History Injection
 100 +               (BHI) vulnerability. Syscalls are hardened against BHI
 101 +               regardless of this setting. This setting affects the deployment
 102 +               of the HW BHI control and the SW BHB clearing sequence.
 103 +
 104 +               on
 105 +                       unconditionally enable.
 106 +               off
 107 +                       unconditionally disable.
 108 +               auto
 109 +                       enable if hardware mitigation
 110 +                       control(BHI_DIS_S) is available.
 111 +
 112  For spectre_v2_user see Documentation/admin-guide/kernel-parameters.txt
 113
 114  Mitigation selection guide
 115 --- a/Documentation/admin-guide/kernel-parameters.txt
 116 +++ b/Documentation/admin-guide/kernel-parameters.txt
 117 @@ -5920,6 +5920,17 @@
 118         sonypi.*=       [HW] Sony Programmable I/O Control Device driver
 119                         See Documentation/admin-guide/laptops/sonypi.rst
 120
 121 +       spectre_bhi=    [X86] Control mitigation of Branch History Injection
 122 +                       (BHI) vulnerability. Syscalls are hardened against BHI
 123 +                       reglardless of this setting. This setting affects the
 124 +                       deployment of the HW BHI control and the SW BHB
 125 +                       clearing sequence.
 126 +
 127 +                       on   - unconditionally enable.
 128 +                       off  - unconditionally disable.
 129 +                       auto - (default) enable only if hardware mitigation
 130 +                              control(BHI_DIS_S) is available.
 131 +
 132         spectre_v2=     [X86] Control mitigation of Spectre variant 2
 133                         (indirect branch speculation) vulnerability.
 134                         The default operation protects the kernel from
 135 --- a/arch/x86/Kconfig
 136 +++ b/arch/x86/Kconfig
 137 @@ -2566,6 +2566,31 @@ config MITIGATION_RFDS
 138           stored in floating point, vector and integer registers.
 139           See also <file:Documentation/admin-guide/hw-vuln/reg-file-data-sampling.rst>
 140
 141 +choice
 142 +       prompt "Clear branch history"
 143 +       depends on CPU_SUP_INTEL
 144 +       default SPECTRE_BHI_AUTO
 145 +       help
 146 +         Enable BHI mitigations. BHI attacks are a form of Spectre V2 attacks
 147 +         where the branch history buffer is poisoned to speculatively steer
 148 +         indirect branches.
 149 +         See <file:Documentation/admin-guide/hw-vuln/spectre.rst>
 150 +
 151 +config SPECTRE_BHI_ON
 152 +       bool "on"
 153 +       help
 154 +         Equivalent to setting spectre_bhi=on command line parameter.
 155 +config SPECTRE_BHI_OFF
 156 +       bool "off"
 157 +       help
 158 +         Equivalent to setting spectre_bhi=off command line parameter.
 159 +config SPECTRE_BHI_AUTO
 160 +       bool "auto"
 161 +       help
 162 +         Equivalent to setting spectre_bhi=auto command line parameter.
 163 +
 164 +endchoice
 165 +
 166  endif
 167
 168  config ARCH_HAS_ADD_PAGES
 169 --- a/arch/x86/include/asm/cpufeatures.h
 170 +++ b/arch/x86/include/asm/cpufeatures.h
 171 @@ -465,6 +465,7 @@
 172  #define X86_FEATURE_AMD_LBR_PMC_FREEZE (21*32+ 0) /* AMD LBR and PMC Freeze */
 173  #define X86_FEATURE_CLEAR_BHB_LOOP     (21*32+ 1) /* "" Clear branch history at syscall entry using SW loop */
 174  #define X86_FEATURE_BHI_CTRL           (21*32+ 2) /* "" BHI_DIS_S HW control available */
 175 +#define X86_FEATURE_CLEAR_BHB_HW       (21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 176
 177  /*
 178   * BUG word(s)
 179 --- a/arch/x86/kernel/cpu/bugs.c
 180 +++ b/arch/x86/kernel/cpu/bugs.c
 181 @@ -1606,6 +1606,74 @@ static void __init spectre_v2_determine_
 182         dump_stack();
 183  }
 184
 185 +/*
 186 + * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by
 187 + * branch history in userspace. Not needed if BHI_NO is set.
 188 + */
 189 +static bool __init spec_ctrl_bhi_dis(void)
 190 +{
 191 +       if (!boot_cpu_has(X86_FEATURE_BHI_CTRL))
 192 +               return false;
 193 +
 194 +       x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S;
 195 +       update_spec_ctrl(x86_spec_ctrl_base);
 196 +       setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW);
 197 +
 198 +       return true;
 199 +}
 200 +
 201 +enum bhi_mitigations {
 202 +       BHI_MITIGATION_OFF,
 203 +       BHI_MITIGATION_ON,
 204 +       BHI_MITIGATION_AUTO,
 205 +};
 206 +
 207 +static enum bhi_mitigations bhi_mitigation __ro_after_init =
 208 +       IS_ENABLED(CONFIG_SPECTRE_BHI_ON)  ? BHI_MITIGATION_ON  :
 209 +       IS_ENABLED(CONFIG_SPECTRE_BHI_OFF) ? BHI_MITIGATION_OFF :
 210 +                                            BHI_MITIGATION_AUTO;
 211 +
 212 +static int __init spectre_bhi_parse_cmdline(char *str)
 213 +{
 214 +       if (!str)
 215 +               return -EINVAL;
 216 +
 217 +       if (!strcmp(str, "off"))
 218 +               bhi_mitigation = BHI_MITIGATION_OFF;
 219 +       else if (!strcmp(str, "on"))
 220 +               bhi_mitigation = BHI_MITIGATION_ON;
 221 +       else if (!strcmp(str, "auto"))
 222 +               bhi_mitigation = BHI_MITIGATION_AUTO;
 223 +       else
 224 +               pr_err("Ignoring unknown spectre_bhi option (%s)", str);
 225 +
 226 +       return 0;
 227 +}
 228 +early_param("spectre_bhi", spectre_bhi_parse_cmdline);
 229 +
 230 +static void __init bhi_select_mitigation(void)
 231 +{
 232 +       if (bhi_mitigation == BHI_MITIGATION_OFF)
 233 +               return;
 234 +
 235 +       /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
 236 +       if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
 237 +           !(x86_read_arch_cap_msr() & ARCH_CAP_RRSBA))
 238 +               return;
 239 +
 240 +       if (spec_ctrl_bhi_dis())
 241 +               return;
 242 +
 243 +       if (!IS_ENABLED(CONFIG_X86_64))
 244 +               return;
 245 +
 246 +       if (bhi_mitigation == BHI_MITIGATION_AUTO)
 247 +               return;
 248 +
 249 +       setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP);
 250 +       pr_info("Spectre BHI mitigation: SW BHB clearing on syscall\n");
 251 +}
 252 +
 253  static void __init spectre_v2_select_mitigation(void)
 254  {
 255         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 256 @@ -1717,6 +1785,9 @@ static void __init spectre_v2_select_mit
 257             mode == SPECTRE_V2_RETPOLINE)
 258                 spec_ctrl_disable_kernel_rrsba();
 259
 260 +       if (boot_cpu_has(X86_BUG_BHI))
 261 +               bhi_select_mitigation();
 262 +
 263         spectre_v2_enabled = mode;
 264         pr_info("%s\n", spectre_v2_strings[mode]);
 265
 266 @@ -2732,6 +2803,21 @@ static char *pbrsb_eibrs_state(void)
 267         }
 268  }
 269
 270 +static const char * const spectre_bhi_state(void)
 271 +{
 272 +       if (!boot_cpu_has_bug(X86_BUG_BHI))
 273 +               return "; BHI: Not affected";
 274 +       else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
 275 +               return "; BHI: BHI_DIS_S";
 276 +       else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
 277 +               return "; BHI: SW loop";
 278 +       else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
 279 +                !(x86_read_arch_cap_msr() & ARCH_CAP_RRSBA))
 280 +               return "; BHI: Retpoline";
 281 +
 282 +       return "; BHI: Vulnerable (Syscall hardening enabled)";
 283 +}
 284 +
 285  static ssize_t spectre_v2_show_state(char *buf)
 286  {
 287         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
 288 @@ -2744,13 +2830,15 @@ static ssize_t spectre_v2_show_state(cha
 289             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
 290                 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
 291
 292 -       return sysfs_emit(buf, "%s%s%s%s%s%s%s\n",
 293 +       return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n",
 294                           spectre_v2_strings[spectre_v2_enabled],
 295                           ibpb_state(),
 296                           boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "",
 297                           stibp_state(),
 298                           boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "",
 299                           pbrsb_eibrs_state(),
 300 +                         spectre_bhi_state(),
 301 +                         /* this should always be at the end */
 302                           spectre_v2_module_string());
 303  }
 304