x86/process: Move the buffer clearing before MONITOR

Commit 8e786a85c0a3c0fffae6244733fb576eeabd9dec upstream.

Move the VERW clearing before the MONITOR so that VERW doesn't disarm it
and the machine never enters C1.

Original idea by Kim Phillips <kim.phillips@amd.com>.

Suggested-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index b409579f5261ed34180554dd3b81946f74cedcda..7f9a97c572fe29a5167d5f5065105179487489e1 100644 (file)
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -44,8 +44,6 @@ static __always_inline void __monitorx(const void *eax, unsigned long ecx,
 
 static __always_inline void __mwait(unsigned long eax, unsigned long ecx)
 {
-       x86_idle_clear_cpu_buffers();
-
        /* "mwait %eax, %ecx;" */
        asm volatile(".byte 0x0f, 0x01, 0xc9;"
                     :: "a" (eax), "c" (ecx));
@@ -98,7 +96,6 @@ static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx,
  */
 static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)
 {
-       x86_idle_clear_cpu_buffers();
 
        /* "mwait %eax, %ecx;" */
        asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
@@ -117,21 +114,29 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)
  */
 static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {
                const void *addr = &current_thread_info()->flags;
 
                alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));
                __monitor(addr, 0, 0);
 
-               if (!need_resched()) {
-                       if (ecx & 1) {
-                               __mwait(eax, ecx);
-                       } else {
-                               __sti_mwait(eax, ecx);
-                               raw_local_irq_disable();
-                       }
+               if (need_resched())
+                       goto out;
+
+               if (ecx & 1) {
+                       __mwait(eax, ecx);
+               } else {
+                       __sti_mwait(eax, ecx);
+                       raw_local_irq_disable();
                }
        }
+
+out:
        current_clr_polling();
 }
 

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 1dbd7a34645c29616b8de8f181f9ef03336c16ad..4c9c98c5deabdacdfab61c56b8e669f7357730f9 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -911,16 +911,24 @@ static __init bool prefer_mwait_c1_over_halt(void)
  */
 static __cpuidle void mwait_idle(void)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (!current_set_polling_and_test()) {
                const void *addr = &current_thread_info()->flags;
 
                alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));
                __monitor(addr, 0, 0);
-               if (!need_resched()) {
-                       __sti_mwait(0, 0);
-                       raw_local_irq_disable();
-               }
+               if (need_resched())
+                       goto out;
+
+               __sti_mwait(0, 0);
+               raw_local_irq_disable();
        }
+
+out:
        __current_clr_polling();
 }