x86/entry_64: Add VERW just before userspace transition

commit 3c7501722e6b31a6e56edd23cea5e77dbb9ffd1a upstream.

Mitigation for MDS is to use VERW instruction to clear any secrets in
CPU Buffers. Any memory accesses after VERW execution can still remain
in CPU buffers. It is safer to execute VERW late in return to user path
to minimize the window in which kernel data can end up in CPU buffers.
There are not many kernel secrets to be had after SWITCH_TO_USER_CR3.

Add support for deploying VERW mitigation after user register state is
restored. This helps minimize the chances of kernel data ending up into
CPU buffers after executing VERW.

Note that the mitigation at the new location is not yet enabled.

  Corner case not handled
  =======================
  Interrupts returning to kernel don't clear CPUs buffers since the
  exit-to-user path is expected to do that anyways. But, there could be
  a case when an NMI is generated in kernel after the exit-to-user path
  has cleared the buffers. This case is not handled and NMI returning to
  kernel don't clear CPU buffers because:

  1. It is rare to get an NMI after VERW, but before returning to user.
  2. For an unprivileged user, there is no known way to make that NMI
     less rare or target it.
  3. It would take a large number of these precisely-timed NMIs to mount
     an actual attack.  There's presumably not enough bandwidth.
  4. The NMI in question occurs after a VERW, i.e. when user state is
     restored and most interesting data is already scrubbed. Whats left
     is only the data that NMI touches, and that may or may not be of
     any interest.

  [ pawan: resolved conflict in syscall_return_via_sysret, added
           CLEAR_CPU_BUFFERS to USERGS_SYSRET64 ]

Suggested-by: Dave Hansen <dave.hansen@intel.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/all/20240213-delay-verw-v8-2-a6216d83edb7%40linux.intel.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 23212c53cef7f2ee35e8ff864fff7b16c1723a1d..1631a9a1566e3ebea6f8b1d8c6377a24e24e970a 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -615,6 +615,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
        /* Restore RDI. */
        popq    %rdi
        SWAPGS
+       CLEAR_CPU_BUFFERS
        INTERRUPT_RETURN
 
 
@@ -721,6 +722,8 @@ native_irq_return_ldt:
         */
        popq    %rax                            /* Restore user RAX */
 
+       CLEAR_CPU_BUFFERS
+
        /*
         * RSP now points to an ordinary IRET frame, except that the page
         * is read-only and RSP[31:16] are preloaded with the userspace
@@ -1487,6 +1490,12 @@ nmi_restore:
        std
        movq    $0, 5*8(%rsp)           /* clear "NMI executing" */
 
+       /*
+        * Skip CLEAR_CPU_BUFFERS here, since it only helps in rare cases like
+        * NMI in kernel after user state is restored. For an unprivileged user
+        * these conditions are hard to meet.
+        */
+
        /*
         * iretq reads the "iret" frame and exits the NMI stack in a
         * single instruction.  We are returning to kernel mode, so this
@@ -1504,6 +1513,7 @@ SYM_CODE_END(asm_exc_nmi)
 SYM_CODE_START(ignore_sysret)
        UNWIND_HINT_EMPTY
        mov     $-ENOSYS, %eax
+       CLEAR_CPU_BUFFERS
        sysretl
 SYM_CODE_END(ignore_sysret)
 #endif

diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 4d637a965efbe5c64b36f68b96600732ffd49154..7f09e7ad3c74d8887709de00a5575a12af9e31ed 100644 (file)
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -319,6 +319,7 @@ sysret32_from_system_call:
        xorl    %r9d, %r9d
        xorl    %r10d, %r10d
        swapgs
+       CLEAR_CPU_BUFFERS
        sysretl
 SYM_CODE_END(entry_SYSCALL_compat)
 

diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index 8c86edefa11508ab3401ead3bcfd757b3834e015..f40dea50dfbf386bacbb329cfdc10102bc53f2ef 100644 (file)
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -134,6 +134,7 @@ static __always_inline unsigned long arch_local_irq_save(void)
 #define INTERRUPT_RETURN       jmp native_iret
 #define USERGS_SYSRET64                                \
        swapgs;                                 \
+       CLEAR_CPU_BUFFERS;                      \
        sysretq;
 #define USERGS_SYSRET32                                \
        swapgs;                                 \