kaiser: kaiser_flush_tlb_on_return_to_user() check PCID

Let kaiser_flush_tlb_on_return_to_user() do the X86_FEATURE_PCID
check, instead of each caller doing it inline first: nobody needs
to optimize for the noPCID case, it's clearer this way, and better
suits later changes.  Replace those no-op X86_CR3_PCID_KERN_FLUSH lines
by a BUILD_BUG_ON() in load_new_mm_cr3(), in case something changes.

(cherry picked from Change-Id: I9b528ed9d7c1ae4a3b4738c2894ee1740b6fb0b9)

Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 63a101f3379ce7fa797fab6dbfda3caac83dccfe..88d849e51d72403608670638f0150e28a5629d0c 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -90,7 +90,7 @@ static inline void __native_flush_tlb(void)
         * back:
         */
        preempt_disable();
-       if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID))
+       if (kaiser_enabled)
                kaiser_flush_tlb_on_return_to_user();
        native_write_cr3(native_read_cr3());
        preempt_enable();
@@ -145,7 +145,7 @@ static inline void __native_flush_tlb_single(unsigned long addr)
         */
 
        if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) {
-               if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID))
+               if (kaiser_enabled)
                        kaiser_flush_tlb_on_return_to_user();
                asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
                return;

diff --git a/arch/x86/mm/kaiser.c b/arch/x86/mm/kaiser.c
index 5033206881a7959078091552c686ad94da057a5c..1e1bed668848947d91f8f866b645f98c61358e38 100644 (file)
--- a/arch/x86/mm/kaiser.c
+++ b/arch/x86/mm/kaiser.c
@@ -432,12 +432,12 @@ void kaiser_setup_pcid(void)
 
 /*
  * Make a note that this cpu will need to flush USER tlb on return to user.
- * Caller checks whether this_cpu_has(X86_FEATURE_PCID) before calling:
- * if cpu does not, then the NOFLUSH bit will never have been set.
+ * If cpu does not have PCID, then the NOFLUSH bit will never have been set.
  */
 void kaiser_flush_tlb_on_return_to_user(void)
 {
-       this_cpu_write(x86_cr3_pcid_user,
+       if (this_cpu_has(X86_FEATURE_PCID))
+               this_cpu_write(x86_cr3_pcid_user,
                        X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET);
 }
 EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user);

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index ebfae14eb40d5f6efabe43105faf772ab2245c82..5e1536b47589c764406642a55dc472c510061da7 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -21,7 +21,7 @@ static void load_new_mm_cr3(pgd_t *pgdir)
 {
        unsigned long new_mm_cr3 = __pa(pgdir);
 
-       if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID)) {
+       if (kaiser_enabled) {
                /*
                 * We reuse the same PCID for different tasks, so we must
                 * flush all the entries for the PCID out when we change tasks.
@@ -32,10 +32,10 @@ static void load_new_mm_cr3(pgd_t *pgdir)
                 * do it here, but can only be used if X86_FEATURE_INVPCID is
                 * available - and many machines support pcid without invpcid.
                 *
-                * The line below is a no-op: X86_CR3_PCID_KERN_FLUSH is now 0;
-                * but keep that line in there in case something changes.
+                * If X86_CR3_PCID_KERN_FLUSH actually added something, then it
+                * would be needed in the write_cr3() below - if PCIDs enabled.
                 */
-               new_mm_cr3 |= X86_CR3_PCID_KERN_FLUSH;
+               BUILD_BUG_ON(X86_CR3_PCID_KERN_FLUSH);
                kaiser_flush_tlb_on_return_to_user();
        }