2 * Copyright (C) 1994 Linus Torvalds
4 * Cyrix stuff, June 1998 by:
5 * - Rafael R. Reilova (moved everything from head.S),
6 * <rreilova@ececs.uc.edu>
7 * - Channing Corn (tests & fixes),
8 * - Andrew D. Balsa (code cleanup).
10 #include <linux/init.h>
11 #include <linux/utsname.h>
12 #include <linux/cpu.h>
13 #include <linux/module.h>
14 #include <linux/nospec.h>
15 #include <linux/prctl.h>
16 #include <linux/sched/smt.h>
18 #include <asm/spec-ctrl.h>
19 #include <asm/cmdline.h>
21 #include <asm/processor.h>
22 #include <asm/processor-flags.h>
25 #include <asm/paravirt.h>
26 #include <asm/alternative.h>
27 #include <asm/hypervisor.h>
28 #include <asm/pgtable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/intel-family.h>
33 static void __init
spectre_v2_select_mitigation(void);
34 static void __init
ssb_select_mitigation(void);
35 static void __init
l1tf_select_mitigation(void);
36 static void __init
mds_select_mitigation(void);
38 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
39 u64 x86_spec_ctrl_base
;
40 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base
);
41 static DEFINE_MUTEX(spec_ctrl_mutex
);
44 * The vendor and possibly platform specific bits which can be modified in
47 static u64 x86_spec_ctrl_mask
= SPEC_CTRL_IBRS
;
50 * AMD specific MSR info for Speculative Store Bypass control.
51 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
53 u64 x86_amd_ls_cfg_base
;
54 u64 x86_amd_ls_cfg_ssbd_mask
;
56 /* Control conditional STIPB in switch_to() */
57 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp
);
58 /* Control conditional IBPB in switch_mm() */
59 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb
);
60 /* Control unconditional IBPB in switch_mm() */
61 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb
);
63 /* Control MDS CPU buffer clear before returning to user space */
64 DEFINE_STATIC_KEY_FALSE(mds_user_clear
);
65 /* Control MDS CPU buffer clear before idling (halt, mwait) */
66 DEFINE_STATIC_KEY_FALSE(mds_idle_clear
);
67 EXPORT_SYMBOL_GPL(mds_idle_clear
);
69 /* For use by asm MDS_CLEAR_CPU_BUFFERS */
70 const u16 mds_clear_cpu_buffers_ds
= __KERNEL_DS
;
74 static double __initdata x
= 4195835.0;
75 static double __initdata y
= 3145727.0;
78 * This used to check for exceptions..
79 * However, it turns out that to support that,
80 * the XMM trap handlers basically had to
81 * be buggy. So let's have a correct XMM trap
82 * handler, and forget about printing out
83 * some status at boot.
85 * We should really only care about bugs here
86 * anyway. Not features.
88 static void __init
check_fpu(void)
95 * trap_init() enabled FXSR and company _before_ testing for FP
98 * Test for the divl bug: http://en.wikipedia.org/wiki/Fdiv_bug
105 "fsubp %%st,%%st(1)\n\t"
110 : "m" (*&x
), "m" (*&y
));
115 set_cpu_bug(&boot_cpu_data
, X86_BUG_FDIV
);
116 pr_warn("Hmm, FPU with FDIV bug\n");
120 #endif /* CONFIG_X86_32 */
122 void __init
check_bugs(void)
126 * Regardless of whether PCID is enumerated, the SDM says
127 * that it can't be enabled in 32-bit mode.
129 setup_clear_cpu_cap(X86_FEATURE_PCID
);
134 if (!IS_ENABLED(CONFIG_SMP
)) {
136 print_cpu_info(&boot_cpu_data
);
140 * Read the SPEC_CTRL MSR to account for reserved bits which may
141 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
142 * init code as it is not enumerated and depends on the family.
144 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
145 rdmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
147 /* Allow STIBP in MSR_SPEC_CTRL if supported */
148 if (boot_cpu_has(X86_FEATURE_STIBP
))
149 x86_spec_ctrl_mask
|= SPEC_CTRL_STIBP
;
151 /* Select the proper spectre mitigation before patching alternatives */
152 spectre_v2_select_mitigation();
155 * Select proper mitigation for any exposure to the Speculative Store
156 * Bypass vulnerability.
158 ssb_select_mitigation();
160 l1tf_select_mitigation();
162 mds_select_mitigation();
166 * Check whether we are able to run this kernel safely on SMP.
168 * - i386 is no longer supported.
169 * - In order to run on anything without a TSC, we need to be
170 * compiled for a i486.
172 if (boot_cpu_data
.x86
< 4)
173 panic("Kernel requires i486+ for 'invlpg' and other features");
175 init_utsname()->machine
[1] =
176 '0' + (boot_cpu_data
.x86
> 6 ? 6 : boot_cpu_data
.x86
);
177 alternative_instructions();
180 * kernel_fpu_begin/end() in check_fpu() relies on the patched
181 * alternative instructions.
185 #else /* CONFIG_X86_64 */
186 alternative_instructions();
189 * Make sure the first 2MB area is not mapped by huge pages
190 * There are typically fixed size MTRRs in there and overlapping
191 * MTRRs into large pages causes slow downs.
193 * Right now we don't do that with gbpages because there seems
194 * very little benefit for that case.
197 set_memory_4k((unsigned long)__va(0), 1);
202 x86_virt_spec_ctrl(u64 guest_spec_ctrl
, u64 guest_virt_spec_ctrl
, bool setguest
)
204 u64 msrval
, guestval
, hostval
= x86_spec_ctrl_base
;
205 struct thread_info
*ti
= current_thread_info();
207 /* Is MSR_SPEC_CTRL implemented ? */
208 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
)) {
210 * Restrict guest_spec_ctrl to supported values. Clear the
211 * modifiable bits in the host base value and or the
212 * modifiable bits from the guest value.
214 guestval
= hostval
& ~x86_spec_ctrl_mask
;
215 guestval
|= guest_spec_ctrl
& x86_spec_ctrl_mask
;
217 /* SSBD controlled in MSR_SPEC_CTRL */
218 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD
) ||
219 static_cpu_has(X86_FEATURE_AMD_SSBD
))
220 hostval
|= ssbd_tif_to_spec_ctrl(ti
->flags
);
222 /* Conditional STIBP enabled? */
223 if (static_branch_unlikely(&switch_to_cond_stibp
))
224 hostval
|= stibp_tif_to_spec_ctrl(ti
->flags
);
226 if (hostval
!= guestval
) {
227 msrval
= setguest
? guestval
: hostval
;
228 wrmsrl(MSR_IA32_SPEC_CTRL
, msrval
);
233 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
234 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
236 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD
) &&
237 !static_cpu_has(X86_FEATURE_VIRT_SSBD
))
241 * If the host has SSBD mitigation enabled, force it in the host's
242 * virtual MSR value. If its not permanently enabled, evaluate
243 * current's TIF_SSBD thread flag.
245 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
))
246 hostval
= SPEC_CTRL_SSBD
;
248 hostval
= ssbd_tif_to_spec_ctrl(ti
->flags
);
250 /* Sanitize the guest value */
251 guestval
= guest_virt_spec_ctrl
& SPEC_CTRL_SSBD
;
253 if (hostval
!= guestval
) {
256 tif
= setguest
? ssbd_spec_ctrl_to_tif(guestval
) :
257 ssbd_spec_ctrl_to_tif(hostval
);
259 speculation_ctrl_update(tif
);
262 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl
);
264 static void x86_amd_ssb_disable(void)
266 u64 msrval
= x86_amd_ls_cfg_base
| x86_amd_ls_cfg_ssbd_mask
;
268 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD
))
269 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL
, SPEC_CTRL_SSBD
);
270 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD
))
271 wrmsrl(MSR_AMD64_LS_CFG
, msrval
);
275 #define pr_fmt(fmt) "MDS: " fmt
277 /* Default mitigation for L1TF-affected CPUs */
278 static enum mds_mitigations mds_mitigation
= MDS_MITIGATION_FULL
;
280 static const char * const mds_strings
[] = {
281 [MDS_MITIGATION_OFF
] = "Vulnerable",
282 [MDS_MITIGATION_FULL
] = "Mitigation: Clear CPU buffers",
283 [MDS_MITIGATION_VMWERV
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
286 static void __init
mds_select_mitigation(void)
288 if (!boot_cpu_has_bug(X86_BUG_MDS
)) {
289 mds_mitigation
= MDS_MITIGATION_OFF
;
293 if (mds_mitigation
== MDS_MITIGATION_FULL
) {
294 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR
))
295 mds_mitigation
= MDS_MITIGATION_VMWERV
;
296 static_branch_enable(&mds_user_clear
);
298 pr_info("%s\n", mds_strings
[mds_mitigation
]);
301 static int __init
mds_cmdline(char *str
)
303 if (!boot_cpu_has_bug(X86_BUG_MDS
))
309 if (!strcmp(str
, "off"))
310 mds_mitigation
= MDS_MITIGATION_OFF
;
311 else if (!strcmp(str
, "full"))
312 mds_mitigation
= MDS_MITIGATION_FULL
;
316 early_param("mds", mds_cmdline
);
319 #define pr_fmt(fmt) "Spectre V2 : " fmt
321 static enum spectre_v2_mitigation spectre_v2_enabled
= SPECTRE_V2_NONE
;
323 static enum spectre_v2_user_mitigation spectre_v2_user
= SPECTRE_V2_USER_NONE
;
326 static bool spectre_v2_bad_module
;
328 bool retpoline_module_ok(bool has_retpoline
)
330 if (spectre_v2_enabled
== SPECTRE_V2_NONE
|| has_retpoline
)
333 pr_err("System may be vulnerable to spectre v2\n");
334 spectre_v2_bad_module
= true;
338 static inline const char *spectre_v2_module_string(void)
340 return spectre_v2_bad_module
? " - vulnerable module loaded" : "";
343 static inline const char *spectre_v2_module_string(void) { return ""; }
346 static inline bool match_option(const char *arg
, int arglen
, const char *opt
)
348 int len
= strlen(opt
);
350 return len
== arglen
&& !strncmp(arg
, opt
, len
);
353 /* The kernel command line selection for spectre v2 */
354 enum spectre_v2_mitigation_cmd
{
357 SPECTRE_V2_CMD_FORCE
,
358 SPECTRE_V2_CMD_RETPOLINE
,
359 SPECTRE_V2_CMD_RETPOLINE_GENERIC
,
360 SPECTRE_V2_CMD_RETPOLINE_AMD
,
363 enum spectre_v2_user_cmd
{
364 SPECTRE_V2_USER_CMD_NONE
,
365 SPECTRE_V2_USER_CMD_AUTO
,
366 SPECTRE_V2_USER_CMD_FORCE
,
367 SPECTRE_V2_USER_CMD_PRCTL
,
368 SPECTRE_V2_USER_CMD_PRCTL_IBPB
,
369 SPECTRE_V2_USER_CMD_SECCOMP
,
370 SPECTRE_V2_USER_CMD_SECCOMP_IBPB
,
373 static const char * const spectre_v2_user_strings
[] = {
374 [SPECTRE_V2_USER_NONE
] = "User space: Vulnerable",
375 [SPECTRE_V2_USER_STRICT
] = "User space: Mitigation: STIBP protection",
376 [SPECTRE_V2_USER_PRCTL
] = "User space: Mitigation: STIBP via prctl",
377 [SPECTRE_V2_USER_SECCOMP
] = "User space: Mitigation: STIBP via seccomp and prctl",
380 static const struct {
382 enum spectre_v2_user_cmd cmd
;
384 } v2_user_options
[] __initdata
= {
385 { "auto", SPECTRE_V2_USER_CMD_AUTO
, false },
386 { "off", SPECTRE_V2_USER_CMD_NONE
, false },
387 { "on", SPECTRE_V2_USER_CMD_FORCE
, true },
388 { "prctl", SPECTRE_V2_USER_CMD_PRCTL
, false },
389 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB
, false },
390 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP
, false },
391 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB
, false },
394 static void __init
spec_v2_user_print_cond(const char *reason
, bool secure
)
396 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
397 pr_info("spectre_v2_user=%s forced on command line.\n", reason
);
400 static enum spectre_v2_user_cmd __init
401 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd
)
407 case SPECTRE_V2_CMD_NONE
:
408 return SPECTRE_V2_USER_CMD_NONE
;
409 case SPECTRE_V2_CMD_FORCE
:
410 return SPECTRE_V2_USER_CMD_FORCE
;
415 ret
= cmdline_find_option(boot_command_line
, "spectre_v2_user",
418 return SPECTRE_V2_USER_CMD_AUTO
;
420 for (i
= 0; i
< ARRAY_SIZE(v2_user_options
); i
++) {
421 if (match_option(arg
, ret
, v2_user_options
[i
].option
)) {
422 spec_v2_user_print_cond(v2_user_options
[i
].option
,
423 v2_user_options
[i
].secure
);
424 return v2_user_options
[i
].cmd
;
428 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg
);
429 return SPECTRE_V2_USER_CMD_AUTO
;
433 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd
)
435 enum spectre_v2_user_mitigation mode
= SPECTRE_V2_USER_NONE
;
436 bool smt_possible
= IS_ENABLED(CONFIG_SMP
);
437 enum spectre_v2_user_cmd cmd
;
439 if (!boot_cpu_has(X86_FEATURE_IBPB
) && !boot_cpu_has(X86_FEATURE_STIBP
))
442 if (!IS_ENABLED(CONFIG_X86_HT
))
443 smt_possible
= false;
445 cmd
= spectre_v2_parse_user_cmdline(v2_cmd
);
447 case SPECTRE_V2_USER_CMD_NONE
:
449 case SPECTRE_V2_USER_CMD_FORCE
:
450 mode
= SPECTRE_V2_USER_STRICT
;
452 case SPECTRE_V2_USER_CMD_PRCTL
:
453 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
454 mode
= SPECTRE_V2_USER_PRCTL
;
456 case SPECTRE_V2_USER_CMD_AUTO
:
457 case SPECTRE_V2_USER_CMD_SECCOMP
:
458 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
459 if (IS_ENABLED(CONFIG_SECCOMP
))
460 mode
= SPECTRE_V2_USER_SECCOMP
;
462 mode
= SPECTRE_V2_USER_PRCTL
;
466 /* Initialize Indirect Branch Prediction Barrier */
467 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
468 setup_force_cpu_cap(X86_FEATURE_USE_IBPB
);
471 case SPECTRE_V2_USER_CMD_FORCE
:
472 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
473 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
474 static_branch_enable(&switch_mm_always_ibpb
);
476 case SPECTRE_V2_USER_CMD_PRCTL
:
477 case SPECTRE_V2_USER_CMD_AUTO
:
478 case SPECTRE_V2_USER_CMD_SECCOMP
:
479 static_branch_enable(&switch_mm_cond_ibpb
);
485 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
486 static_key_enabled(&switch_mm_always_ibpb
) ?
487 "always-on" : "conditional");
490 /* If enhanced IBRS is enabled no STIPB required */
491 if (spectre_v2_enabled
== SPECTRE_V2_IBRS_ENHANCED
)
495 * If SMT is not possible or STIBP is not available clear the STIPB
498 if (!smt_possible
|| !boot_cpu_has(X86_FEATURE_STIBP
))
499 mode
= SPECTRE_V2_USER_NONE
;
501 spectre_v2_user
= mode
;
502 /* Only print the STIBP mode when SMT possible */
504 pr_info("%s\n", spectre_v2_user_strings
[mode
]);
507 static const char * const spectre_v2_strings
[] = {
508 [SPECTRE_V2_NONE
] = "Vulnerable",
509 [SPECTRE_V2_RETPOLINE_MINIMAL
] = "Vulnerable: Minimal generic ASM retpoline",
510 [SPECTRE_V2_RETPOLINE_MINIMAL_AMD
] = "Vulnerable: Minimal AMD ASM retpoline",
511 [SPECTRE_V2_RETPOLINE_GENERIC
] = "Mitigation: Full generic retpoline",
512 [SPECTRE_V2_RETPOLINE_AMD
] = "Mitigation: Full AMD retpoline",
513 [SPECTRE_V2_IBRS_ENHANCED
] = "Mitigation: Enhanced IBRS",
516 static const struct {
518 enum spectre_v2_mitigation_cmd cmd
;
520 } mitigation_options
[] __initdata
= {
521 { "off", SPECTRE_V2_CMD_NONE
, false },
522 { "on", SPECTRE_V2_CMD_FORCE
, true },
523 { "retpoline", SPECTRE_V2_CMD_RETPOLINE
, false },
524 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD
, false },
525 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC
, false },
526 { "auto", SPECTRE_V2_CMD_AUTO
, false },
529 static void __init
spec_v2_print_cond(const char *reason
, bool secure
)
531 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
532 pr_info("%s selected on command line.\n", reason
);
535 static inline bool retp_compiler(void)
537 return __is_defined(RETPOLINE
);
540 static enum spectre_v2_mitigation_cmd __init
spectre_v2_parse_cmdline(void)
542 enum spectre_v2_mitigation_cmd cmd
= SPECTRE_V2_CMD_AUTO
;
546 if (cmdline_find_option_bool(boot_command_line
, "nospectre_v2"))
547 return SPECTRE_V2_CMD_NONE
;
549 ret
= cmdline_find_option(boot_command_line
, "spectre_v2", arg
, sizeof(arg
));
551 return SPECTRE_V2_CMD_AUTO
;
553 for (i
= 0; i
< ARRAY_SIZE(mitigation_options
); i
++) {
554 if (!match_option(arg
, ret
, mitigation_options
[i
].option
))
556 cmd
= mitigation_options
[i
].cmd
;
560 if (i
>= ARRAY_SIZE(mitigation_options
)) {
561 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
562 return SPECTRE_V2_CMD_AUTO
;
565 if ((cmd
== SPECTRE_V2_CMD_RETPOLINE
||
566 cmd
== SPECTRE_V2_CMD_RETPOLINE_AMD
||
567 cmd
== SPECTRE_V2_CMD_RETPOLINE_GENERIC
) &&
568 !IS_ENABLED(CONFIG_RETPOLINE
)) {
569 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options
[i
].option
);
570 return SPECTRE_V2_CMD_AUTO
;
573 if (cmd
== SPECTRE_V2_CMD_RETPOLINE_AMD
&&
574 boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
) {
575 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
576 return SPECTRE_V2_CMD_AUTO
;
579 spec_v2_print_cond(mitigation_options
[i
].option
,
580 mitigation_options
[i
].secure
);
584 static void __init
spectre_v2_select_mitigation(void)
586 enum spectre_v2_mitigation_cmd cmd
= spectre_v2_parse_cmdline();
587 enum spectre_v2_mitigation mode
= SPECTRE_V2_NONE
;
590 * If the CPU is not affected and the command line mode is NONE or AUTO
591 * then nothing to do.
593 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) &&
594 (cmd
== SPECTRE_V2_CMD_NONE
|| cmd
== SPECTRE_V2_CMD_AUTO
))
598 case SPECTRE_V2_CMD_NONE
:
601 case SPECTRE_V2_CMD_FORCE
:
602 case SPECTRE_V2_CMD_AUTO
:
603 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED
)) {
604 mode
= SPECTRE_V2_IBRS_ENHANCED
;
605 /* Force it so VMEXIT will restore correctly */
606 x86_spec_ctrl_base
|= SPEC_CTRL_IBRS
;
607 wrmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
608 goto specv2_set_mode
;
610 if (IS_ENABLED(CONFIG_RETPOLINE
))
613 case SPECTRE_V2_CMD_RETPOLINE_AMD
:
614 if (IS_ENABLED(CONFIG_RETPOLINE
))
617 case SPECTRE_V2_CMD_RETPOLINE_GENERIC
:
618 if (IS_ENABLED(CONFIG_RETPOLINE
))
619 goto retpoline_generic
;
621 case SPECTRE_V2_CMD_RETPOLINE
:
622 if (IS_ENABLED(CONFIG_RETPOLINE
))
626 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
630 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
) {
632 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC
)) {
633 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
634 goto retpoline_generic
;
636 mode
= retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD
:
637 SPECTRE_V2_RETPOLINE_MINIMAL_AMD
;
638 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD
);
639 setup_force_cpu_cap(X86_FEATURE_RETPOLINE
);
642 mode
= retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC
:
643 SPECTRE_V2_RETPOLINE_MINIMAL
;
644 setup_force_cpu_cap(X86_FEATURE_RETPOLINE
);
648 spectre_v2_enabled
= mode
;
649 pr_info("%s\n", spectre_v2_strings
[mode
]);
652 * If spectre v2 protection has been enabled, unconditionally fill
653 * RSB during a context switch; this protects against two independent
656 * - RSB underflow (and switch to BTB) on Skylake+
657 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
659 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW
);
660 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
663 * Retpoline means the kernel is safe because it has no indirect
664 * branches. Enhanced IBRS protects firmware too, so, enable restricted
665 * speculation around firmware calls only when Enhanced IBRS isn't
668 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
669 * the user might select retpoline on the kernel command line and if
670 * the CPU supports Enhanced IBRS, kernel might un-intentionally not
671 * enable IBRS around firmware calls.
673 if (boot_cpu_has(X86_FEATURE_IBRS
) && mode
!= SPECTRE_V2_IBRS_ENHANCED
) {
674 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW
);
675 pr_info("Enabling Restricted Speculation for firmware calls\n");
678 /* Set up IBPB and STIBP depending on the general spectre V2 command */
679 spectre_v2_user_select_mitigation(cmd
);
681 /* Enable STIBP if appropriate */
685 static void update_stibp_msr(void * __unused
)
687 wrmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
690 /* Update x86_spec_ctrl_base in case SMT state changed. */
691 static void update_stibp_strict(void)
693 u64 mask
= x86_spec_ctrl_base
& ~SPEC_CTRL_STIBP
;
695 if (sched_smt_active())
696 mask
|= SPEC_CTRL_STIBP
;
698 if (mask
== x86_spec_ctrl_base
)
701 pr_info("Update user space SMT mitigation: STIBP %s\n",
702 mask
& SPEC_CTRL_STIBP
? "always-on" : "off");
703 x86_spec_ctrl_base
= mask
;
704 on_each_cpu(update_stibp_msr
, NULL
, 1);
707 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
708 static void update_indir_branch_cond(void)
710 if (sched_smt_active())
711 static_branch_enable(&switch_to_cond_stibp
);
713 static_branch_disable(&switch_to_cond_stibp
);
716 /* Update the static key controlling the MDS CPU buffer clear in idle */
717 static void update_mds_branch_idle(void)
720 * Enable the idle clearing if SMT is active on CPUs which are
721 * affected only by MSBDS and not any other MDS variant.
723 * The other variants cannot be mitigated when SMT is enabled, so
724 * clearing the buffers on idle just to prevent the Store Buffer
725 * repartitioning leak would be a window dressing exercise.
727 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY
))
730 if (sched_smt_active())
731 static_branch_enable(&mds_idle_clear
);
733 static_branch_disable(&mds_idle_clear
);
736 void arch_smt_update(void)
738 /* Enhanced IBRS implies STIBP. No update required. */
739 if (spectre_v2_enabled
== SPECTRE_V2_IBRS_ENHANCED
)
742 mutex_lock(&spec_ctrl_mutex
);
744 switch (spectre_v2_user
) {
745 case SPECTRE_V2_USER_NONE
:
747 case SPECTRE_V2_USER_STRICT
:
748 update_stibp_strict();
750 case SPECTRE_V2_USER_PRCTL
:
751 case SPECTRE_V2_USER_SECCOMP
:
752 update_indir_branch_cond();
756 switch (mds_mitigation
) {
757 case MDS_MITIGATION_FULL
:
758 case MDS_MITIGATION_VMWERV
:
759 update_mds_branch_idle();
761 case MDS_MITIGATION_OFF
:
765 mutex_unlock(&spec_ctrl_mutex
);
769 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt
771 static enum ssb_mitigation ssb_mode
= SPEC_STORE_BYPASS_NONE
;
773 /* The kernel command line selection */
774 enum ssb_mitigation_cmd
{
775 SPEC_STORE_BYPASS_CMD_NONE
,
776 SPEC_STORE_BYPASS_CMD_AUTO
,
777 SPEC_STORE_BYPASS_CMD_ON
,
778 SPEC_STORE_BYPASS_CMD_PRCTL
,
779 SPEC_STORE_BYPASS_CMD_SECCOMP
,
782 static const char * const ssb_strings
[] = {
783 [SPEC_STORE_BYPASS_NONE
] = "Vulnerable",
784 [SPEC_STORE_BYPASS_DISABLE
] = "Mitigation: Speculative Store Bypass disabled",
785 [SPEC_STORE_BYPASS_PRCTL
] = "Mitigation: Speculative Store Bypass disabled via prctl",
786 [SPEC_STORE_BYPASS_SECCOMP
] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
789 static const struct {
791 enum ssb_mitigation_cmd cmd
;
792 } ssb_mitigation_options
[] __initdata
= {
793 { "auto", SPEC_STORE_BYPASS_CMD_AUTO
}, /* Platform decides */
794 { "on", SPEC_STORE_BYPASS_CMD_ON
}, /* Disable Speculative Store Bypass */
795 { "off", SPEC_STORE_BYPASS_CMD_NONE
}, /* Don't touch Speculative Store Bypass */
796 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL
}, /* Disable Speculative Store Bypass via prctl */
797 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP
}, /* Disable Speculative Store Bypass via prctl and seccomp */
800 static enum ssb_mitigation_cmd __init
ssb_parse_cmdline(void)
802 enum ssb_mitigation_cmd cmd
= SPEC_STORE_BYPASS_CMD_AUTO
;
806 if (cmdline_find_option_bool(boot_command_line
, "nospec_store_bypass_disable")) {
807 return SPEC_STORE_BYPASS_CMD_NONE
;
809 ret
= cmdline_find_option(boot_command_line
, "spec_store_bypass_disable",
812 return SPEC_STORE_BYPASS_CMD_AUTO
;
814 for (i
= 0; i
< ARRAY_SIZE(ssb_mitigation_options
); i
++) {
815 if (!match_option(arg
, ret
, ssb_mitigation_options
[i
].option
))
818 cmd
= ssb_mitigation_options
[i
].cmd
;
822 if (i
>= ARRAY_SIZE(ssb_mitigation_options
)) {
823 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
824 return SPEC_STORE_BYPASS_CMD_AUTO
;
831 static enum ssb_mitigation __init
__ssb_select_mitigation(void)
833 enum ssb_mitigation mode
= SPEC_STORE_BYPASS_NONE
;
834 enum ssb_mitigation_cmd cmd
;
836 if (!boot_cpu_has(X86_FEATURE_SSBD
))
839 cmd
= ssb_parse_cmdline();
840 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
) &&
841 (cmd
== SPEC_STORE_BYPASS_CMD_NONE
||
842 cmd
== SPEC_STORE_BYPASS_CMD_AUTO
))
846 case SPEC_STORE_BYPASS_CMD_AUTO
:
847 case SPEC_STORE_BYPASS_CMD_SECCOMP
:
849 * Choose prctl+seccomp as the default mode if seccomp is
852 if (IS_ENABLED(CONFIG_SECCOMP
))
853 mode
= SPEC_STORE_BYPASS_SECCOMP
;
855 mode
= SPEC_STORE_BYPASS_PRCTL
;
857 case SPEC_STORE_BYPASS_CMD_ON
:
858 mode
= SPEC_STORE_BYPASS_DISABLE
;
860 case SPEC_STORE_BYPASS_CMD_PRCTL
:
861 mode
= SPEC_STORE_BYPASS_PRCTL
;
863 case SPEC_STORE_BYPASS_CMD_NONE
:
868 * We have three CPU feature flags that are in play here:
869 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
870 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
871 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
873 if (mode
== SPEC_STORE_BYPASS_DISABLE
) {
874 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
);
876 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
877 * use a completely different MSR and bit dependent on family.
879 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD
) &&
880 !static_cpu_has(X86_FEATURE_AMD_SSBD
)) {
881 x86_amd_ssb_disable();
883 x86_spec_ctrl_base
|= SPEC_CTRL_SSBD
;
884 x86_spec_ctrl_mask
|= SPEC_CTRL_SSBD
;
885 wrmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
892 static void ssb_select_mitigation(void)
894 ssb_mode
= __ssb_select_mitigation();
896 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
897 pr_info("%s\n", ssb_strings
[ssb_mode
]);
901 #define pr_fmt(fmt) "Speculation prctl: " fmt
903 static void task_update_spec_tif(struct task_struct
*tsk
)
905 /* Force the update of the real TIF bits */
906 set_tsk_thread_flag(tsk
, TIF_SPEC_FORCE_UPDATE
);
909 * Immediately update the speculation control MSRs for the current
910 * task, but for a non-current task delay setting the CPU
911 * mitigation until it is scheduled next.
913 * This can only happen for SECCOMP mitigation. For PRCTL it's
914 * always the current task.
917 speculation_ctrl_update_current();
920 static int ssb_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
922 if (ssb_mode
!= SPEC_STORE_BYPASS_PRCTL
&&
923 ssb_mode
!= SPEC_STORE_BYPASS_SECCOMP
)
928 /* If speculation is force disabled, enable is not allowed */
929 if (task_spec_ssb_force_disable(task
))
931 task_clear_spec_ssb_disable(task
);
932 task_update_spec_tif(task
);
934 case PR_SPEC_DISABLE
:
935 task_set_spec_ssb_disable(task
);
936 task_update_spec_tif(task
);
938 case PR_SPEC_FORCE_DISABLE
:
939 task_set_spec_ssb_disable(task
);
940 task_set_spec_ssb_force_disable(task
);
941 task_update_spec_tif(task
);
949 static int ib_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
953 if (spectre_v2_user
== SPECTRE_V2_USER_NONE
)
956 * Indirect branch speculation is always disabled in strict
959 if (spectre_v2_user
== SPECTRE_V2_USER_STRICT
)
961 task_clear_spec_ib_disable(task
);
962 task_update_spec_tif(task
);
964 case PR_SPEC_DISABLE
:
965 case PR_SPEC_FORCE_DISABLE
:
967 * Indirect branch speculation is always allowed when
968 * mitigation is force disabled.
970 if (spectre_v2_user
== SPECTRE_V2_USER_NONE
)
972 if (spectre_v2_user
== SPECTRE_V2_USER_STRICT
)
974 task_set_spec_ib_disable(task
);
975 if (ctrl
== PR_SPEC_FORCE_DISABLE
)
976 task_set_spec_ib_force_disable(task
);
977 task_update_spec_tif(task
);
985 int arch_prctl_spec_ctrl_set(struct task_struct
*task
, unsigned long which
,
989 case PR_SPEC_STORE_BYPASS
:
990 return ssb_prctl_set(task
, ctrl
);
991 case PR_SPEC_INDIRECT_BRANCH
:
992 return ib_prctl_set(task
, ctrl
);
998 #ifdef CONFIG_SECCOMP
999 void arch_seccomp_spec_mitigate(struct task_struct
*task
)
1001 if (ssb_mode
== SPEC_STORE_BYPASS_SECCOMP
)
1002 ssb_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1003 if (spectre_v2_user
== SPECTRE_V2_USER_SECCOMP
)
1004 ib_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1008 static int ssb_prctl_get(struct task_struct
*task
)
1011 case SPEC_STORE_BYPASS_DISABLE
:
1012 return PR_SPEC_DISABLE
;
1013 case SPEC_STORE_BYPASS_SECCOMP
:
1014 case SPEC_STORE_BYPASS_PRCTL
:
1015 if (task_spec_ssb_force_disable(task
))
1016 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1017 if (task_spec_ssb_disable(task
))
1018 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1019 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1021 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
1022 return PR_SPEC_ENABLE
;
1023 return PR_SPEC_NOT_AFFECTED
;
1027 static int ib_prctl_get(struct task_struct
*task
)
1029 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
))
1030 return PR_SPEC_NOT_AFFECTED
;
1032 switch (spectre_v2_user
) {
1033 case SPECTRE_V2_USER_NONE
:
1034 return PR_SPEC_ENABLE
;
1035 case SPECTRE_V2_USER_PRCTL
:
1036 case SPECTRE_V2_USER_SECCOMP
:
1037 if (task_spec_ib_force_disable(task
))
1038 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1039 if (task_spec_ib_disable(task
))
1040 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1041 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1042 case SPECTRE_V2_USER_STRICT
:
1043 return PR_SPEC_DISABLE
;
1045 return PR_SPEC_NOT_AFFECTED
;
1049 int arch_prctl_spec_ctrl_get(struct task_struct
*task
, unsigned long which
)
1052 case PR_SPEC_STORE_BYPASS
:
1053 return ssb_prctl_get(task
);
1054 case PR_SPEC_INDIRECT_BRANCH
:
1055 return ib_prctl_get(task
);
1061 void x86_spec_ctrl_setup_ap(void)
1063 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
1064 wrmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
1066 if (ssb_mode
== SPEC_STORE_BYPASS_DISABLE
)
1067 x86_amd_ssb_disable();
1071 #define pr_fmt(fmt) "L1TF: " fmt
1073 * These CPUs all support 44bits physical address space internally in the
1074 * cache but CPUID can report a smaller number of physical address bits.
1076 * The L1TF mitigation uses the top most address bit for the inversion of
1077 * non present PTEs. When the installed memory reaches into the top most
1078 * address bit due to memory holes, which has been observed on machines
1079 * which report 36bits physical address bits and have 32G RAM installed,
1080 * then the mitigation range check in l1tf_select_mitigation() triggers.
1081 * This is a false positive because the mitigation is still possible due to
1082 * the fact that the cache uses 44bit internally. Use the cache bits
1083 * instead of the reported physical bits and adjust them on the affected
1084 * machines to 44bit if the reported bits are less than 44.
1086 static void override_cache_bits(struct cpuinfo_x86
*c
)
1091 switch (c
->x86_model
) {
1092 case INTEL_FAM6_NEHALEM
:
1093 case INTEL_FAM6_WESTMERE
:
1094 case INTEL_FAM6_SANDYBRIDGE
:
1095 case INTEL_FAM6_IVYBRIDGE
:
1096 case INTEL_FAM6_HASWELL_CORE
:
1097 case INTEL_FAM6_HASWELL_ULT
:
1098 case INTEL_FAM6_HASWELL_GT3E
:
1099 case INTEL_FAM6_BROADWELL_CORE
:
1100 case INTEL_FAM6_BROADWELL_GT3E
:
1101 case INTEL_FAM6_SKYLAKE_MOBILE
:
1102 case INTEL_FAM6_SKYLAKE_DESKTOP
:
1103 case INTEL_FAM6_KABYLAKE_MOBILE
:
1104 case INTEL_FAM6_KABYLAKE_DESKTOP
:
1105 if (c
->x86_cache_bits
< 44)
1106 c
->x86_cache_bits
= 44;
1111 static void __init
l1tf_select_mitigation(void)
1115 if (!boot_cpu_has_bug(X86_BUG_L1TF
))
1118 override_cache_bits(&boot_cpu_data
);
1120 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1121 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1125 half_pa
= (u64
)l1tf_pfn_limit() << PAGE_SHIFT
;
1126 if (e820_any_mapped(half_pa
, ULLONG_MAX
- half_pa
, E820_RAM
)) {
1127 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1128 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1130 pr_info("However, doing so will make a part of your RAM unusable.\n");
1131 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1135 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV
);
1141 static ssize_t
mds_show_state(char *buf
)
1143 #ifdef CONFIG_HYPERVISOR_GUEST
1145 return sprintf(buf
, "%s; SMT Host state unknown\n",
1146 mds_strings
[mds_mitigation
]);
1150 if (boot_cpu_has(X86_BUG_MSBDS_ONLY
)) {
1151 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
1152 sched_smt_active() ? "mitigated" : "disabled");
1155 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
1156 sched_smt_active() ? "vulnerable" : "disabled");
1159 static char *stibp_state(void)
1161 if (spectre_v2_enabled
== SPECTRE_V2_IBRS_ENHANCED
)
1164 switch (spectre_v2_user
) {
1165 case SPECTRE_V2_USER_NONE
:
1166 return ", STIBP: disabled";
1167 case SPECTRE_V2_USER_STRICT
:
1168 return ", STIBP: forced";
1169 case SPECTRE_V2_USER_PRCTL
:
1170 case SPECTRE_V2_USER_SECCOMP
:
1171 if (static_key_enabled(&switch_to_cond_stibp
))
1172 return ", STIBP: conditional";
1177 static char *ibpb_state(void)
1179 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
1180 if (static_key_enabled(&switch_mm_always_ibpb
))
1181 return ", IBPB: always-on";
1182 if (static_key_enabled(&switch_mm_cond_ibpb
))
1183 return ", IBPB: conditional";
1184 return ", IBPB: disabled";
1189 static ssize_t
cpu_show_common(struct device
*dev
, struct device_attribute
*attr
,
1190 char *buf
, unsigned int bug
)
1192 if (!boot_cpu_has_bug(bug
))
1193 return sprintf(buf
, "Not affected\n");
1196 case X86_BUG_CPU_MELTDOWN
:
1197 if (boot_cpu_has(X86_FEATURE_KAISER
))
1198 return sprintf(buf
, "Mitigation: PTI\n");
1202 case X86_BUG_SPECTRE_V1
:
1203 return sprintf(buf
, "Mitigation: __user pointer sanitization\n");
1205 case X86_BUG_SPECTRE_V2
:
1206 return sprintf(buf
, "%s%s%s%s%s%s\n", spectre_v2_strings
[spectre_v2_enabled
],
1208 boot_cpu_has(X86_FEATURE_USE_IBRS_FW
) ? ", IBRS_FW" : "",
1210 boot_cpu_has(X86_FEATURE_RSB_CTXSW
) ? ", RSB filling" : "",
1211 spectre_v2_module_string());
1213 case X86_BUG_SPEC_STORE_BYPASS
:
1214 return sprintf(buf
, "%s\n", ssb_strings
[ssb_mode
]);
1217 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV
))
1218 return sprintf(buf
, "Mitigation: Page Table Inversion\n");
1222 return mds_show_state(buf
);
1228 return sprintf(buf
, "Vulnerable\n");
1231 ssize_t
cpu_show_meltdown(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1233 return cpu_show_common(dev
, attr
, buf
, X86_BUG_CPU_MELTDOWN
);
1236 ssize_t
cpu_show_spectre_v1(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1238 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V1
);
1241 ssize_t
cpu_show_spectre_v2(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1243 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V2
);
1246 ssize_t
cpu_show_spec_store_bypass(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1248 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPEC_STORE_BYPASS
);
1251 ssize_t
cpu_show_l1tf(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1253 return cpu_show_common(dev
, attr
, buf
, X86_BUG_L1TF
);
1256 ssize_t
cpu_show_mds(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1258 return cpu_show_common(dev
, attr
, buf
, X86_BUG_MDS
);