arm64: mm: Wrap flush_tlb_page() around __do_flush_tlb_range()

Flushing a page from the tlb is just a special case of flushing a range.
So let's rework flush_tlb_page() so that it simply wraps
__do_flush_tlb_range(). While at it, let's also update the API to take
the same flags that we use when flushing a range. This allows us to
delete all the ugly "_nosync", "_local" and "_nonotify" variants.

Thanks to constant folding, all of the complex looping and tlbi-by-range
options get eliminated so that the generated code for flush_tlb_page()
looks very similar to the previous version.

Reviewed-by: Linu Cherian <linu.cherian@arm.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 88bb9275ac898a47e451f72da6e80146c31c074c..7039931df462272074c8ea51236dca9a6f4e3249 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -101,10 +101,10 @@ static inline void arch_leave_lazy_mmu_mode(void)
  * entries exist.
  */
 #define flush_tlb_fix_spurious_fault(vma, address, ptep)       \
-       local_flush_tlb_page_nonotify(vma, address)
+       __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
 
 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)   \
-       local_flush_tlb_page_nonotify(vma, address)
+       __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
 
 /*
  * ZERO_PAGE is a global shared page that is always zero: used
@@ -1320,7 +1320,7 @@ static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
                 * context-switch, which provides a DSB to complete the TLB
                 * invalidation.
                 */
-               flush_tlb_page_nosync(vma, address);
+               __flush_tlb_page(vma, address, TLBF_NOSYNC);
        }
 
        return young;

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 5509927e45b93b4a6ace6c2e4e1f86d6919d61be..5096ec7ab8650e3d57876050e19992ab4416d7d7 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -269,10 +269,7 @@ static inline void __tlbi_sync_s1ish_hyp(void)
  *             unmapping pages from vmalloc/io space.
  *
  *     flush_tlb_page(vma, addr)
- *             Invalidate a single user mapping for address 'addr' in the
- *             address space corresponding to 'vma->mm'.  Note that this
- *             operation only invalidates a single, last-level page-table
- *             entry and therefore does not affect any walk-caches.
+ *             Equivalent to __flush_tlb_page(..., flags=TLBF_NONE)
  *
  *
  *     Next, we have some undocumented invalidation routines that you probably
@@ -300,13 +297,14 @@ static inline void __tlbi_sync_s1ish_hyp(void)
  *             TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
  *             (only perform the invalidation for the local cpu).
  *
- *     local_flush_tlb_page(vma, addr)
- *             Local variant of flush_tlb_page().  Stale TLB entries may
- *             remain in remote CPUs.
- *
- *     local_flush_tlb_page_nonotify(vma, addr)
- *             Same as local_flush_tlb_page() except MMU notifier will not be
- *             called.
+ *     __flush_tlb_page(vma, addr, flags)
+ *             Invalidate a single user mapping for address 'addr' in the
+ *             address space corresponding to 'vma->mm'.  Note that this
+ *             operation only invalidates a single, last-level page-table entry
+ *             and therefore does not affect any walk-caches. flags may contain
+ *             any combination of TLBF_NONOTIFY (don't call mmu notifiers),
+ *             TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
+ *             (only perform the invalidation for the local cpu).
  *
  *     Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
  *     on top of these routines, since that is our interface to the mmu_gather
@@ -340,51 +338,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
        mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
 }
 
-static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,
-                                                         unsigned long uaddr)
-{
-       dsb(nshst);
-       __tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));
-}
-
-static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,
-                                                unsigned long uaddr)
-{
-       __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
-       dsb(nsh);
-}
-
-static inline void local_flush_tlb_page(struct vm_area_struct *vma,
-                                       unsigned long uaddr)
-{
-       __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
-       mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,
-                                               (uaddr & PAGE_MASK) + PAGE_SIZE);
-       dsb(nsh);
-}
-
-static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
-                                          unsigned long uaddr)
-{
-       dsb(ishst);
-       __tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));
-       mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
-                                               (uaddr & PAGE_MASK) + PAGE_SIZE);
-}
-
-static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
-                                        unsigned long uaddr)
-{
-       return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
-}
-
-static inline void flush_tlb_page(struct vm_area_struct *vma,
-                                 unsigned long uaddr)
-{
-       flush_tlb_page_nosync(vma, uaddr);
-       __tlbi_sync_s1ish();
-}
-
 static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
 {
        return true;
@@ -632,6 +585,22 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
        __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);
 }
 
+static inline void __flush_tlb_page(struct vm_area_struct *vma,
+                                   unsigned long uaddr, tlbf_t flags)
+{
+       unsigned long start = round_down(uaddr, PAGE_SIZE);
+       unsigned long end = start + PAGE_SIZE;
+
+       __do_flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN,
+                            TLBF_NOWALKCACHE | flags);
+}
+
+static inline void flush_tlb_page(struct vm_area_struct *vma,
+                                 unsigned long uaddr)
+{
+       __flush_tlb_page(vma, uaddr, TLBF_NONE);
+}
+
 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
        const unsigned long stride = PAGE_SIZE;

diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index be9dab2c7d6a8e8335550b9e2c46d4bc935108fc..f91aa686f1428bbbe099ff19961d603983c74254 100644 (file)
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -239,7 +239,7 @@ int __ptep_set_access_flags(struct vm_area_struct *vma,
         * flush_tlb_fix_spurious_fault().
         */
        if (dirty)
-               local_flush_tlb_page(vma, address);
+               __flush_tlb_page(vma, address, TLBF_NOBROADCAST);
        return 1;
 }