x86/process: Move the buffer clearing before MONITOR

From: "Borislav Petkov (AMD)" <bp@alien8.de>

Commit 8e786a85c0a3c0fffae6244733fb576eeabd9dec upstream.

Move the VERW clearing before the MONITOR so that VERW doesn't disarm it
and the machine never enters C1.

Original idea by Kim Phillips <kim.phillips@amd.com>.

Suggested-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index 20b33e6370c31802472d3a2b07df2637c0ee5538..2a2de4f3cb204e7fae74e094786ee3e1461344f1 100644 (file)
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -43,8 +43,6 @@ static inline void __monitorx(const void *eax, unsigned long ecx,
 
 static inline void __mwait(unsigned long eax, unsigned long ecx)
 {
-       x86_idle_clear_cpu_buffers();
-
        /* "mwait %eax, %ecx;" */
        asm volatile(".byte 0x0f, 0x01, 0xc9;"
                     :: "a" (eax), "c" (ecx));
@@ -88,7 +86,6 @@ static inline void __mwaitx(unsigned long eax, unsigned long ebx,
 
 static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
 {
-       x86_idle_clear_cpu_buffers();
 
        /* "mwait %eax, %ecx;" */
        asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
@@ -107,6 +104,11 @@ static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
  */
 static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {
                if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) {
                        mb();
@@ -115,9 +117,13 @@ static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
                }
 
                __monitor((void *)&current_thread_info()->flags, 0, 0);
-               if (!need_resched())
-                       __mwait(eax, ecx);
+               if (need_resched())
+                       goto out;
+
+               __mwait(eax, ecx);
        }
+
+out:
        current_clr_polling();
 }
 

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 34e7c49b8057dbc3775e65a9a8638be6eb7d5d15..fd91c301218d5f0b383a5e1c23a5934ca6125d4f 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -825,6 +825,11 @@ static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
  */
 static __cpuidle void mwait_idle(void)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (!current_set_polling_and_test()) {
                if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
                        mb(); /* quirk */
@@ -833,13 +838,17 @@ static __cpuidle void mwait_idle(void)
                }
 
                __monitor((void *)&current_thread_info()->flags, 0, 0);
-               if (!need_resched())
-                       __sti_mwait(0, 0);
-               else
+               if (need_resched()) {
                        raw_local_irq_enable();
+                       goto out;
+               }
+
+               __sti_mwait(0, 0);
        } else {
                raw_local_irq_enable();
        }
+
+out:
        __current_clr_polling();
 }