x86/entry_32: Add VERW just before userspace transition

As done for entry_64, add support for executing VERW late in exit to
user path for 32-bit mode.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/all/20240213-delay-verw-v8-3-a6216d83edb7%40linux.intel.com

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index c73047bf9f4bff9c4631c0eab383cedceda41918..fba427646805d55221664538be2285c3ae188ca1 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -885,6 +885,7 @@ SYM_FUNC_START(entry_SYSENTER_32)
        BUG_IF_WRONG_CR3 no_user_check=1
        popfl
        popl    %eax
+       CLEAR_CPU_BUFFERS
 
        /*
         * Return back to the vDSO, which will pop ecx and edx.
@@ -954,6 +955,7 @@ restore_all_switch_stack:
 
        /* Restore user state */
        RESTORE_REGS pop=4                      # skip orig_eax/error_code
+       CLEAR_CPU_BUFFERS
 .Lirq_return:
        /*
         * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
@@ -1146,6 +1148,7 @@ SYM_CODE_START(asm_exc_nmi)
 
        /* Not on SYSENTER stack. */
        call    exc_nmi
+       CLEAR_CPU_BUFFERS
        jmp     .Lnmi_return
 
 .Lnmi_from_sysenter_stack: