arm64: mm: Remove pmd_sect() and pud_sect()

The semantics of pXd_leaf() are very similar to pXd_sect(). The only
difference is that pXd_sect() only considers it a section if PTE_VALID
is set, whereas pXd_leaf() permits both "valid" and "present-invalid"
types.

Using pXd_sect() has caused issues now that large leaf entries can be
present-invalid since commit a166563e7ec37 ("arm64: mm: support large
block mapping when rodata=full"), so let's just remove the API and
standardize on pXd_leaf().

There are a few callsites of the form pXd_leaf(READ_ONCE(*pXdp)). This
was previously fine for the pXd_sect() macro because it only evaluated
its argument once. But pXd_leaf() evaluates its argument multiple times.
So let's avoid unintended side effects by reimplementing pXd_leaf() as
an inline function.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index dd062179b9b66acdda9194f4b84f6b3a1f705692..5bc42b85acfc0c8019182e0414e98a007b47ec30 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -784,9 +784,13 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 
 #define pmd_table(pmd)         ((pmd_val(pmd) & PMD_TYPE_MASK) == \
                                 PMD_TYPE_TABLE)
-#define pmd_sect(pmd)          ((pmd_val(pmd) & PMD_TYPE_MASK) == \
-                                PMD_TYPE_SECT)
-#define pmd_leaf(pmd)          (pmd_present(pmd) && !pmd_table(pmd))
+
+#define pmd_leaf pmd_leaf
+static inline bool pmd_leaf(pmd_t pmd)
+{
+       return pmd_present(pmd) && !pmd_table(pmd);
+}
+
 #define pmd_bad(pmd)           (!pmd_table(pmd))
 
 #define pmd_leaf_size(pmd)     (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
@@ -804,11 +808,8 @@ static inline int pmd_trans_huge(pmd_t pmd)
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
-static inline bool pud_sect(pud_t pud) { return false; }
 static inline bool pud_table(pud_t pud) { return true; }
 #else
-#define pud_sect(pud)          ((pud_val(pud) & PUD_TYPE_MASK) == \
-                                PUD_TYPE_SECT)
 #define pud_table(pud)         ((pud_val(pud) & PUD_TYPE_MASK) == \
                                 PUD_TYPE_TABLE)
 #endif
@@ -878,7 +879,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
                                 PUD_TYPE_TABLE)
 #define pud_present(pud)       pte_present(pud_pte(pud))
 #ifndef __PAGETABLE_PMD_FOLDED
-#define pud_leaf(pud)          (pud_present(pud) && !pud_table(pud))
+#define pud_leaf pud_leaf
+static inline bool pud_leaf(pud_t pud)
+{
+       return pud_present(pud) && !pud_table(pud);
+}
 #else
 #define pud_leaf(pud)          false
 #endif

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 1575680675d8dedb3076d15c92175f85638584c4..dcee56bb622adbef8dbd4840cccf19421acc9a0b 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -204,7 +204,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
        pmd_t pmd = READ_ONCE(*pmdp);
        pte_t *ptep;
 
-       BUG_ON(pmd_sect(pmd));
+       BUG_ON(pmd_leaf(pmd));
        if (pmd_none(pmd)) {
                pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF;
                phys_addr_t pte_phys;
@@ -303,7 +303,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
        /*
         * Check for initial section mappings in the pgd/pud.
         */
-       BUG_ON(pud_sect(pud));
+       BUG_ON(pud_leaf(pud));
        if (pud_none(pud)) {
                pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF;
                phys_addr_t pmd_phys;
@@ -1503,7 +1503,7 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr,
                        continue;
 
                WARN_ON(!pmd_present(pmd));
-               if (pmd_sect(pmd)) {
+               if (pmd_leaf(pmd)) {
                        pmd_clear(pmdp);
 
                        /*
@@ -1536,7 +1536,7 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr,
                        continue;
 
                WARN_ON(!pud_present(pud));
-               if (pud_sect(pud)) {
+               if (pud_leaf(pud)) {
                        pud_clear(pudp);
 
                        /*
@@ -1650,7 +1650,7 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
                if (pmd_none(pmd))
                        continue;
 
-               WARN_ON(!pmd_present(pmd) || !pmd_table(pmd) || pmd_sect(pmd));
+               WARN_ON(!pmd_present(pmd) || !pmd_table(pmd));
                free_empty_pte_table(pmdp, addr, next, floor, ceiling);
        } while (addr = next, addr < end);
 
@@ -1690,7 +1690,7 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
                if (pud_none(pud))
                        continue;
 
-               WARN_ON(!pud_present(pud) || !pud_table(pud) || pud_sect(pud));
+               WARN_ON(!pud_present(pud) || !pud_table(pud));
                free_empty_pmd_table(pudp, addr, next, floor, ceiling);
        } while (addr = next, addr < end);
 
@@ -1786,7 +1786,7 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
 {
        vmemmap_verify((pte_t *)pmdp, node, addr, next);
 
-       return pmd_sect(READ_ONCE(*pmdp));
+       return pmd_leaf(READ_ONCE(*pmdp));
 }
 
 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
@@ -1850,7 +1850,7 @@ void p4d_clear_huge(p4d_t *p4dp)
 
 int pud_clear_huge(pud_t *pudp)
 {
-       if (!pud_sect(READ_ONCE(*pudp)))
+       if (!pud_leaf(READ_ONCE(*pudp)))
                return 0;
        pud_clear(pudp);
        return 1;
@@ -1858,7 +1858,7 @@ int pud_clear_huge(pud_t *pudp)
 
 int pmd_clear_huge(pmd_t *pmdp)
 {
-       if (!pmd_sect(READ_ONCE(*pmdp)))
+       if (!pmd_leaf(READ_ONCE(*pmdp)))
                return 0;
        pmd_clear(pmdp);
        return 1;