releases/5.0.16/0009-x86-speculation-mds-Conditionally-clear-CPU-buffers-.patch

   1 From c13d5738b6681167eaab57fbf66fa20b9ee819c8 Mon Sep 17 00:00:00 2001
   2 From: Thomas Gleixner <tglx@linutronix.de>
   3 Date: Mon, 18 Feb 2019 23:04:01 +0100
   4 Subject: [PATCH 09/27] x86/speculation/mds: Conditionally clear CPU buffers on
   5  idle entry
   6
   7 commit 07f07f55a29cb705e221eda7894dd67ab81ef343 upstream
   8
   9 Add a static key which controls the invocation of the CPU buffer clear
  10 mechanism on idle entry. This is independent of other MDS mitigations
  11 because the idle entry invocation to mitigate the potential leakage due to
  12 store buffer repartitioning is only necessary on SMT systems.
  13
  14 Add the actual invocations to the different halt/mwait variants which
  15 covers all usage sites. mwaitx is not patched as it's not available on
  16 Intel CPUs.
  17
  18 The buffer clear is only invoked before entering the C-State to prevent
  19 that stale data from the idling CPU is spilled to the Hyper-Thread sibling
  20 after the Store buffer got repartitioned and all entries are available to
  21 the non idle sibling.
  22
  23 When coming out of idle the store buffer is partitioned again so each
  24 sibling has half of it available. Now CPU which returned from idle could be
  25 speculatively exposed to contents of the sibling, but the buffers are
  26 flushed either on exit to user space or on VMENTER.
  27
  28 When later on conditional buffer clearing is implemented on top of this,
  29 then there is no action required either because before returning to user
  30 space the context switch will set the condition flag which causes a flush
  31 on the return to user path.
  32
  33 Note, that the buffer clearing on idle is only sensible on CPUs which are
  34 solely affected by MSBDS and not any other variant of MDS because the other
  35 MDS variants cannot be mitigated when SMT is enabled, so the buffer
  36 clearing on idle would be a window dressing exercise.
  37
  38 This intentionally does not handle the case in the acpi/processor_idle
  39 driver which uses the legacy IO port interface for C-State transitions for
  40 two reasons:
  41
  42  - The acpi/processor_idle driver was replaced by the intel_idle driver
  43    almost a decade ago. Anything Nehalem upwards supports it and defaults
  44    to that new driver.
  45
  46  - The legacy IO port interface is likely to be used on older and therefore
  47    unaffected CPUs or on systems which do not receive microcode updates
  48    anymore, so there is no point in adding that.
  49
  50 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  51 Reviewed-by: Borislav Petkov <bp@suse.de>
  52 Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  53 Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
  54 Reviewed-by: Jon Masters <jcm@redhat.com>
  55 Tested-by: Jon Masters <jcm@redhat.com>
  56 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  57 ---
  58  Documentation/x86/mds.rst            | 42 ++++++++++++++++++++++++++++
  59  arch/x86/include/asm/irqflags.h      |  4 +++
  60  arch/x86/include/asm/mwait.h         |  7 +++++
  61  arch/x86/include/asm/nospec-branch.h | 12 ++++++++
  62  arch/x86/kernel/cpu/bugs.c           |  3 ++
  63  5 files changed, 68 insertions(+)
  64
  65 diff --git a/Documentation/x86/mds.rst b/Documentation/x86/mds.rst
  66 index 54d935bf283b..87ce8ac9f36e 100644
  67 --- a/Documentation/x86/mds.rst
  68 +++ b/Documentation/x86/mds.rst
  69 @@ -149,3 +149,45 @@ Mitigation points
  70       This takes the paranoid exit path only when the INT1 breakpoint is in
  71       kernel space. #DB on a user space address takes the regular exit path,
  72       so no extra mitigation required.
  73 +
  74 +
  75 +2. C-State transition
  76 +^^^^^^^^^^^^^^^^^^^^^
  77 +
  78 +   When a CPU goes idle and enters a C-State the CPU buffers need to be
  79 +   cleared on affected CPUs when SMT is active. This addresses the
  80 +   repartitioning of the store buffer when one of the Hyper-Threads enters
  81 +   a C-State.
  82 +
  83 +   When SMT is inactive, i.e. either the CPU does not support it or all
  84 +   sibling threads are offline CPU buffer clearing is not required.
  85 +
  86 +   The idle clearing is enabled on CPUs which are only affected by MSBDS
  87 +   and not by any other MDS variant. The other MDS variants cannot be
  88 +   protected against cross Hyper-Thread attacks because the Fill Buffer and
  89 +   the Load Ports are shared. So on CPUs affected by other variants, the
  90 +   idle clearing would be a window dressing exercise and is therefore not
  91 +   activated.
  92 +
  93 +   The invocation is controlled by the static key mds_idle_clear which is
  94 +   switched depending on the chosen mitigation mode and the SMT state of
  95 +   the system.
  96 +
  97 +   The buffer clear is only invoked before entering the C-State to prevent
  98 +   that stale data from the idling CPU from spilling to the Hyper-Thread
  99 +   sibling after the store buffer got repartitioned and all entries are
 100 +   available to the non idle sibling.
 101 +
 102 +   When coming out of idle the store buffer is partitioned again so each
 103 +   sibling has half of it available. The back from idle CPU could be then
 104 +   speculatively exposed to contents of the sibling. The buffers are
 105 +   flushed either on exit to user space or on VMENTER so malicious code
 106 +   in user space or the guest cannot speculatively access them.
 107 +
 108 +   The mitigation is hooked into all variants of halt()/mwait(), but does
 109 +   not cover the legacy ACPI IO-Port mechanism because the ACPI idle driver
 110 +   has been superseded by the intel_idle driver around 2010 and is
 111 +   preferred on all affected CPUs which are expected to gain the MD_CLEAR
 112 +   functionality in microcode. Aside of that the IO-Port mechanism is a
 113 +   legacy interface which is only used on older systems which are either
 114 +   not affected or do not receive microcode updates anymore.
 115 diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
 116 index 058e40fed167..8a0e56e1dcc9 100644
 117 --- a/arch/x86/include/asm/irqflags.h
 118 +++ b/arch/x86/include/asm/irqflags.h
 119 @@ -6,6 +6,8 @@
 120
 121  #ifndef __ASSEMBLY__
 122
 123 +#include <asm/nospec-branch.h>
 124 +
 125  /* Provide __cpuidle; we can't safely include <linux/cpu.h> */
 126  #define __cpuidle __attribute__((__section__(".cpuidle.text")))
 127
 128 @@ -54,11 +56,13 @@ static inline void native_irq_enable(void)
 129
 130  static inline __cpuidle void native_safe_halt(void)
 131  {
 132 +       mds_idle_clear_cpu_buffers();
 133         asm volatile("sti; hlt": : :"memory");
 134  }
 135
 136  static inline __cpuidle void native_halt(void)
 137  {
 138 +       mds_idle_clear_cpu_buffers();
 139         asm volatile("hlt": : :"memory");
 140  }
 141
 142 diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
 143 index 39a2fb29378a..eb0f80ce8524 100644
 144 --- a/arch/x86/include/asm/mwait.h
 145 +++ b/arch/x86/include/asm/mwait.h
 146 @@ -6,6 +6,7 @@
 147  #include <linux/sched/idle.h>
 148
 149  #include <asm/cpufeature.h>
 150 +#include <asm/nospec-branch.h>
 151
 152  #define MWAIT_SUBSTATE_MASK            0xf
 153  #define MWAIT_CSTATE_MASK              0xf
 154 @@ -40,6 +41,8 @@ static inline void __monitorx(const void *eax, unsigned long ecx,
 155
 156  static inline void __mwait(unsigned long eax, unsigned long ecx)
 157  {
 158 +       mds_idle_clear_cpu_buffers();
 159 +
 160         /* "mwait %eax, %ecx;" */
 161         asm volatile(".byte 0x0f, 0x01, 0xc9;"
 162                      :: "a" (eax), "c" (ecx));
 163 @@ -74,6 +77,8 @@ static inline void __mwait(unsigned long eax, unsigned long ecx)
 164  static inline void __mwaitx(unsigned long eax, unsigned long ebx,
 165                             unsigned long ecx)
 166  {
 167 +       /* No MDS buffer clear as this is AMD/HYGON only */
 168 +
 169         /* "mwaitx %eax, %ebx, %ecx;" */
 170         asm volatile(".byte 0x0f, 0x01, 0xfb;"
 171                      :: "a" (eax), "b" (ebx), "c" (ecx));
 172 @@ -81,6 +86,8 @@ static inline void __mwaitx(unsigned long eax, unsigned long ebx,
 173
 174  static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
 175  {
 176 +       mds_idle_clear_cpu_buffers();
 177 +
 178         trace_hardirqs_on();
 179         /* "mwait %eax, %ecx;" */
 180         asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
 181 diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
 182 index 65b747286d96..4e970390110f 100644
 183 --- a/arch/x86/include/asm/nospec-branch.h
 184 +++ b/arch/x86/include/asm/nospec-branch.h
 185 @@ -319,6 +319,7 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 186  DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 187
 188  DECLARE_STATIC_KEY_FALSE(mds_user_clear);
 189 +DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
 190
 191  #include <asm/segment.h>
 192
 193 @@ -356,6 +357,17 @@ static inline void mds_user_clear_cpu_buffers(void)
 194                 mds_clear_cpu_buffers();
 195  }
 196
 197 +/**
 198 + * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
 199 + *
 200 + * Clear CPU buffers if the corresponding static key is enabled
 201 + */
 202 +static inline void mds_idle_clear_cpu_buffers(void)
 203 +{
 204 +       if (static_branch_likely(&mds_idle_clear))
 205 +               mds_clear_cpu_buffers();
 206 +}
 207 +
 208  #endif /* __ASSEMBLY__ */
 209
 210  /*
 211 diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
 212 index bef397b4c2f8..10d309f99f11 100644
 213 --- a/arch/x86/kernel/cpu/bugs.c
 214 +++ b/arch/x86/kernel/cpu/bugs.c
 215 @@ -66,6 +66,9 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 216  /* Control MDS CPU buffer clear before returning to user space */
 217  DEFINE_STATIC_KEY_FALSE(mds_user_clear);
 218  EXPORT_SYMBOL_GPL(mds_user_clear);
 219 +/* Control MDS CPU buffer clear before idling (halt, mwait) */
 220 +DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
 221 +EXPORT_SYMBOL_GPL(mds_idle_clear);
 222
 223  void __init check_bugs(void)
 224  {
 225 --
 226 2.21.0
 227