arm64: mm: Refactor __flush_tlb_range() to take flags

We have function variants with "_nosync", "_local", "_nonotify" as well
as the "last_level" parameter. Let's generalize and simplify by using a
flags parameter to encode all these variants.

As a first step, convert the "last_level" boolean parameter to a flags
parameter and create the first flag, TLBF_NOWALKCACHE. When present,
walk cache entries are not evicted, which is the same as the old
last_level=true.

Reviewed-by: Linu Cherian <linu.cherian@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
index e6f8ff3cc6306e98e513ab107d1ef3c6c352a934..d038ff14d16cae967d2c3d33363ee2a263fe62bd 100644 (file)
--- a/arch/arm64/include/asm/hugetlb.h
+++ b/arch/arm64/include/asm/hugetlb.h
@@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma,
                                             unsigned long start,
                                             unsigned long end,
                                             unsigned long stride,
-                                            bool last_level)
+                                            tlbf_t flags)
 {
        switch (stride) {
 #ifndef __PAGETABLE_PMD_FOLDED
        case PUD_SIZE:
-               __flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);
+               __flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);
                break;
 #endif
        case CONT_PMD_SIZE:
        case PMD_SIZE:
-               __flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);
+               __flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);
                break;
        case CONT_PTE_SIZE:
-               __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);
+               __flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);
                break;
        default:
-               __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);
+               __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);
        }
 }
 
@@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
 {
        unsigned long stride = huge_page_size(hstate_vma(vma));
 
-       __flush_hugetlb_tlb_range(vma, start, end, stride, false);
+       __flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);
 }
 
 #endif /* __ASM_HUGETLB_H */

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index b3e58735c49bdd1cc86542663727cb8a029482e3..88bb9275ac898a47e451f72da6e80146c31c074c 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -89,9 +89,9 @@ static inline void arch_leave_lazy_mmu_mode(void)
 
 /* Set stride and tlb_level in flush_*_tlb_range */
 #define flush_pmd_tlb_range(vma, addr, end)    \
-       __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
+       __flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)
 #define flush_pud_tlb_range(vma, addr, end)    \
-       __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
+       __flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 /*

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 8d762607285cc49610b68212da7456fdc64603d0..10869d7731b83deae446ad0aa99618024a257559 100644 (file)
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -53,7 +53,7 @@ static inline int tlb_get_level(struct mmu_gather *tlb)
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
        struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
-       bool last_level = !tlb->freed_tables;
+       tlbf_t flags = tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE;
        unsigned long stride = tlb_get_unmap_size(tlb);
        int tlb_level = tlb_get_level(tlb);
 
@@ -63,13 +63,13 @@ static inline void tlb_flush(struct mmu_gather *tlb)
         * reallocate our ASID without invalidating the entire TLB.
         */
        if (tlb->fullmm) {
-               if (!last_level)
+               if (tlb->freed_tables)
                        flush_tlb_mm(tlb->mm);
                return;
        }
 
        __flush_tlb_range(&vma, tlb->start, tlb->end, stride,
-                         last_level, tlb_level);
+                         tlb_level, flags);
 }
 
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 0a49a25a4fdc8e12e625e0b009ecd10fe427ad64..d134824ea5daaf09bbb500e14f1e4a1651c92830 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -286,16 +286,16 @@ static inline void __tlbi_sync_s1ish_hyp(void)
  *             CPUs, ensuring that any walk-cache entries associated with the
  *             translation are also invalidated.
  *
- *     __flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
+ *     __flush_tlb_range(vma, start, end, stride, tlb_level, flags)
  *             Invalidate the virtual-address range '[start, end)' on all
  *             CPUs for the user address space corresponding to 'vma->mm'.
  *             The invalidation operations are issued at a granularity
- *             determined by 'stride' and only affect any walk-cache entries
- *             if 'last_level' is equal to false. tlb_level is the level at
+ *             determined by 'stride'. tlb_level is the level at
  *             which the invalidation must take place. If the level is wrong,
  *             no invalidation may take place. In the case where the level
  *             cannot be easily determined, the value TLBI_TTL_UNKNOWN will
- *             perform a non-hinted invalidation.
+ *             perform a non-hinted invalidation. flags may be TLBF_NONE (0) or
+ *             TLBF_NOWALKCACHE (elide eviction of walk cache entries).
  *
  *     local_flush_tlb_page(vma, addr)
  *             Local variant of flush_tlb_page().  Stale TLB entries may
@@ -544,10 +544,18 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
        return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
 }
 
+typedef unsigned __bitwise tlbf_t;
+
+/* No special behaviour. */
+#define TLBF_NONE              ((__force tlbf_t)0)
+
+/* Invalidate tlb entries only, leaving the page table walk cache intact. */
+#define TLBF_NOWALKCACHE       ((__force tlbf_t)BIT(0))
+
 static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
                                     unsigned long start, unsigned long end,
-                                    unsigned long stride, bool last_level,
-                                    int tlb_level)
+                                    unsigned long stride, int tlb_level,
+                                    tlbf_t flags)
 {
        unsigned long asid, pages;
 
@@ -563,7 +571,7 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
        dsb(ishst);
        asid = ASID(mm);
 
-       if (last_level)
+       if (flags & TLBF_NOWALKCACHE)
                __flush_s1_tlb_range_op(vale1is, start, pages, stride,
                                     asid, tlb_level);
        else
@@ -575,11 +583,11 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
 
 static inline void __flush_tlb_range(struct vm_area_struct *vma,
                                     unsigned long start, unsigned long end,
-                                    unsigned long stride, bool last_level,
-                                    int tlb_level)
+                                    unsigned long stride, int tlb_level,
+                                    tlbf_t flags)
 {
        __flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
-                                last_level, tlb_level);
+                                tlb_level, flags);
        __tlbi_sync_s1ish();
 }
 
@@ -607,7 +615,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
         * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
         * information here.
         */
-       __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
+       __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);
 }
 
 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -648,7 +656,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
                struct mm_struct *mm, unsigned long start, unsigned long end)
 {
-       __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
+       __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE);
 }
 
 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)

diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c
index 1519d090d5ea7b586daebba8e0a616690bef56f4..5f2a7b43ca329bbeff08482490777fa7224a60e2 100644 (file)
--- a/arch/arm64/mm/contpte.c
+++ b/arch/arm64/mm/contpte.c
@@ -225,7 +225,8 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
         */
 
        if (!system_supports_bbml2_noabort())
-               __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3);
+               __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3,
+                                 TLBF_NOWALKCACHE);
 
        __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES);
 }
@@ -552,7 +553,7 @@ int contpte_clear_flush_young_ptes(struct vm_area_struct *vma,
                 * eliding the trailing DSB applies here.
                 */
                __flush_tlb_range_nosync(vma->vm_mm, addr, end,
-                                        PAGE_SIZE, true, 3);
+                                        PAGE_SIZE, 3, TLBF_NOWALKCACHE);
        }
 
        return young;

diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index a42c05cf564082a79ce7d0699033aca1ab7ee8f6..0b7ccd0cbb9ec5ecafc582a455de13ed3cf37cc2 100644 (file)
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -181,7 +181,7 @@ static pte_t get_clear_contig_flush(struct mm_struct *mm,
        struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
        unsigned long end = addr + (pgsize * ncontig);
 
-       __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, true);
+       __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, TLBF_NOWALKCACHE);
        return orig_pte;
 }
 
@@ -209,7 +209,7 @@ static void clear_flush(struct mm_struct *mm,
        if (mm == &init_mm)
                flush_tlb_kernel_range(saddr, addr);
        else
-               __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true);
+               __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, TLBF_NOWALKCACHE);
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index a6a00accf4f9380ca17fa3be2aee6abf17009e12..054df431846fd23601d6bd7d94776e25f0261069 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -2149,7 +2149,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
                 */
                if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte))
                        __flush_tlb_range(vma, addr, nr * PAGE_SIZE,
-                                         PAGE_SIZE, true, 3);
+                                         PAGE_SIZE, 3, TLBF_NOWALKCACHE);
        }
 
        return pte;