x86/tlb: Move __flush_tlb_one_user() out of line

cpu_tlbstate is exported because various TLB-related functions need access
to it, but cpu_tlbstate is sensitive information which should only be
accessed by well-contained kernel functions and not be directly exposed to
modules.

As a third step, move _flush_tlb_one_user() out of line and hide the
native function. The latter can be static when CONFIG_PARAVIRT is
disabled.

Consolidate the name space while at it and remove the pointless extra
wrapper in the paravirt code.

No functional change.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200421092559.428213098@linutronix.de

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 712e059bc7c6ed94cc84c189838de7e813de8297..dcd6517a694a1eab23ad12097092d428dd6b501a 100644 (file)
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -49,6 +49,7 @@ static inline void slow_down_io(void)
 
 void native_flush_tlb_local(void);
 void native_flush_tlb_global(void);
+void native_flush_tlb_one_user(unsigned long addr);
 
 static inline void __flush_tlb_local(void)
 {

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index d66d16e3fd67b3410db7c9ad37e76f3528290ed2..14c5b98aaa51afaffb9dea8278ae95519d157031 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -142,11 +142,10 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
 
 void flush_tlb_local(void);
 void flush_tlb_global(void);
+void flush_tlb_one_user(unsigned long addr);
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
-#else
-#define __flush_tlb_one_user(addr)     __native_flush_tlb_one_user(addr)
 #endif
 
 struct tlb_context {
@@ -345,54 +344,6 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask)
 
 extern void initialize_tlbstate_and_flush(void);
 
-/*
- * Given an ASID, flush the corresponding user ASID.  We can delay this
- * until the next time we switch to it.
- *
- * See SWITCH_TO_USER_CR3.
- */
-static inline void invalidate_user_asid(u16 asid)
-{
-       /* There is no user ASID if address space separation is off */
-       if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
-               return;
-
-       /*
-        * We only have a single ASID if PCID is off and the CR3
-        * write will have flushed it.
-        */
-       if (!cpu_feature_enabled(X86_FEATURE_PCID))
-               return;
-
-       if (!static_cpu_has(X86_FEATURE_PTI))
-               return;
-
-       __set_bit(kern_pcid(asid),
-                 (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
-}
-
-/*
- * flush one page in the user mapping
- */
-static inline void __native_flush_tlb_one_user(unsigned long addr)
-{
-       u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
-
-       asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
-
-       if (!static_cpu_has(X86_FEATURE_PTI))
-               return;
-
-       /*
-        * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
-        * Just use invalidate_user_asid() in case we are called early.
-        */
-       if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
-               invalidate_user_asid(loaded_mm_asid);
-       else
-               invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
-}
-
 /*
  * flush everything
  */
@@ -432,7 +383,7 @@ static inline void __flush_tlb_one_kernel(unsigned long addr)
         * kernel address space and for its usermode counterpart, but it does
         * not flush it for other address spaces.
         */
-       __flush_tlb_one_user(addr);
+       flush_tlb_one_user(addr);
 
        if (!static_cpu_has(X86_FEATURE_PTI))
                return;

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 6094b007979c9bfdaee89144b25fa6f4128bb108..5638e4ae2ea6eb552664396f769d57d0b8cf2580 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -160,11 +160,6 @@ unsigned paravirt_patch_insns(void *insn_buff, unsigned len,
        return insn_len;
 }
 
-static void native_flush_tlb_one_user(unsigned long addr)
-{
-       __native_flush_tlb_one_user(addr);
-}
-
 struct static_key paravirt_steal_enabled;
 struct static_key paravirt_steal_rq_enabled;
 

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d548b98e5a49b8b7e95c3d6f7796ab9b6036cb7c..2822602ce60a680740080170d24ac65272b98cb9 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -24,6 +24,7 @@
 # define STATIC_NOPV                   static
 # define __flush_tlb_local             native_flush_tlb_local
 # define __flush_tlb_global            native_flush_tlb_global
+# define __flush_tlb_one_user(addr)    native_flush_tlb_one_user(addr)
 #endif
 
 /*
@@ -118,6 +119,32 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
        *need_flush = true;
 }
 
+/*
+ * Given an ASID, flush the corresponding user ASID.  We can delay this
+ * until the next time we switch to it.
+ *
+ * See SWITCH_TO_USER_CR3.
+ */
+static inline void invalidate_user_asid(u16 asid)
+{
+       /* There is no user ASID if address space separation is off */
+       if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
+               return;
+
+       /*
+        * We only have a single ASID if PCID is off and the CR3
+        * write will have flushed it.
+        */
+       if (!cpu_feature_enabled(X86_FEATURE_PCID))
+               return;
+
+       if (!static_cpu_has(X86_FEATURE_PTI))
+               return;
+
+       __set_bit(kern_pcid(asid),
+                 (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
+}
+
 static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
 {
        unsigned long new_mm_cr3;
@@ -645,7 +672,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
                unsigned long addr = f->start;
 
                while (addr < f->end) {
-                       __flush_tlb_one_user(addr);
+                       flush_tlb_one_user(addr);
                        addr += 1UL << f->stride_shift;
                }
                if (local)
@@ -891,6 +918,33 @@ unsigned long __get_current_cr3_fast(void)
 }
 EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
 
+/*
+ * Flush one page in the user mapping
+ */
+STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
+{
+       u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+
+       asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
+
+       if (!static_cpu_has(X86_FEATURE_PTI))
+               return;
+
+       /*
+        * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
+        * Just use invalidate_user_asid() in case we are called early.
+        */
+       if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
+               invalidate_user_asid(loaded_mm_asid);
+       else
+               invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
+}
+
+void flush_tlb_one_user(unsigned long addr)
+{
+       __flush_tlb_one_user(addr);
+}
+
 /*
  * Flush everything
  */

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 6af766c47dd26bca1a9260e0babf2585bb076d13..4ea69690c3e49a6afbfa2035cb81b831362a3963 100644 (file)
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -296,7 +296,7 @@ static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp,
                flush_tlb_local();
                stat->d_alltlb++;
        } else {
-               __flush_tlb_one_user(msg->address);
+               flush_tlb_one_user(msg->address);
                stat->d_onetlb++;
        }
        stat->d_requestee++;