powerpc/8xx: Reduce alignment constraint for kernel memory

8xx has three large page sizes: 8M, 512k and 16k.

A too big alignment can lead to wasting memory. On a board which has
only 32 MBytes of RAM, every single byte is worth it and a 512k
alignment is sometimes too much.

Allow mapping kernel memory with 16k pages and reduce the constraint
on kernel memory alignment. 512k and 16k pages are handled the same
way so reverse tests in order to make 8M pages the special case and
other ones (512k and 16k) the alternative.

Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/fa9927b70df13627cdf10b992ea71d6562c7760e.1746191262.git.christophe.leroy@csgroup.eu

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 2519fc3538c676ea3d7620938c48dd9c3cb4ece8..c3e0cc83f1205e2bc72ff3dbc6a89cc0a47af960 100644 (file)
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -895,7 +895,7 @@ config DATA_SHIFT
        int "Data shift" if DATA_SHIFT_BOOL
        default 24 if STRICT_KERNEL_RWX && PPC64
        range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32
-       range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx
+       range 14 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx
        range 20 24 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_85xx
        default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
        default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32
@@ -908,10 +908,10 @@ config DATA_SHIFT
          On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO.
          Smaller is the alignment, greater is the number of necessary DBATs.
 
-         On 8xx, large pages (512kb or 8M) are used to map kernel linear
-         memory. Aligning to 8M reduces TLB misses as only 8M pages are used
-         in that case. If PIN_TLB is selected, it must be aligned to 8M as
-         8M pages will be pinned.
+         On 8xx, large pages (16kb or 512kb or 8M) are used to map kernel
+         linear memory. Aligning to 8M reduces TLB misses as only 8M pages
+         are used in that case. If PIN_TLB is selected, it must be aligned
+         to 8M as 8M pages will be pinned.
 
 config ARCH_FORCE_MAX_ORDER
        int "Order of maximal physically contiguous allocations"

diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c
index 8b54f12d1889b75780f15d3d4ef14accff4bc73a..ab1505cf42bf5959dd98d127dd0e07363815333e 100644 (file)
--- a/arch/powerpc/mm/nohash/8xx.c
+++ b/arch/powerpc/mm/nohash/8xx.c
@@ -54,20 +54,13 @@ static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
 {
        pmd_t *pmdp = pmd_off_k(va);
        pte_t *ptep;
-
-       if (WARN_ON(psize != MMU_PAGE_512K && psize != MMU_PAGE_8M))
-               return -EINVAL;
+       unsigned int shift = mmu_psize_to_shift(psize);
 
        if (new) {
                if (WARN_ON(slab_is_available()))
                        return -EINVAL;
 
-               if (psize == MMU_PAGE_512K) {
-                       ptep = early_pte_alloc_kernel(pmdp, va);
-                       /* The PTE should never be already present */
-                       if (WARN_ON(pte_present(*ptep) && pgprot_val(prot)))
-                               return -EINVAL;
-               } else {
+               if (psize == MMU_PAGE_8M) {
                        if (WARN_ON(!pmd_none(*pmdp) || !pmd_none(*(pmdp + 1))))
                                return -EINVAL;
 
@@ -78,20 +71,25 @@ static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
                        pmd_populate_kernel(&init_mm, pmdp + 1, ptep);
 
                        ptep = (pte_t *)pmdp;
+               } else {
+                       ptep = early_pte_alloc_kernel(pmdp, va);
+                       /* The PTE should never be already present */
+                       if (WARN_ON(pte_present(*ptep) && pgprot_val(prot)))
+                               return -EINVAL;
                }
        } else {
-               if (psize == MMU_PAGE_512K)
-                       ptep = pte_offset_kernel(pmdp, va);
-               else
+               if (psize == MMU_PAGE_8M)
                        ptep = (pte_t *)pmdp;
+               else
+                       ptep = pte_offset_kernel(pmdp, va);
        }
 
        if (WARN_ON(!ptep))
                return -ENOMEM;
 
        set_huge_pte_at(&init_mm, va, ptep,
-                       pte_mkhuge(pfn_pte(pa >> PAGE_SHIFT, prot)),
-                       1UL << mmu_psize_to_shift(psize));
+                       arch_make_huge_pte(pfn_pte(pa >> PAGE_SHIFT, prot), shift, 0),
+                       1UL << shift);
 
        return 0;
 }
@@ -123,14 +121,18 @@ static int mmu_mapin_ram_chunk(unsigned long offset, unsigned long top,
        unsigned long p = offset;
        int err = 0;
 
-       WARN_ON(!IS_ALIGNED(offset, SZ_512K) || !IS_ALIGNED(top, SZ_512K));
+       WARN_ON(!IS_ALIGNED(offset, SZ_16K) || !IS_ALIGNED(top, SZ_16K));
 
+       for (; p < ALIGN(p, SZ_512K) && p < top && !err; p += SZ_16K, v += SZ_16K)
+               err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_16K, new);
        for (; p < ALIGN(p, SZ_8M) && p < top && !err; p += SZ_512K, v += SZ_512K)
                err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_512K, new);
        for (; p < ALIGN_DOWN(top, SZ_8M) && p < top && !err; p += SZ_8M, v += SZ_8M)
                err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_8M, new);
        for (; p < ALIGN_DOWN(top, SZ_512K) && p < top && !err; p += SZ_512K, v += SZ_512K)
                err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_512K, new);
+       for (; p < ALIGN_DOWN(top, SZ_16K) && p < top && !err; p += SZ_16K, v += SZ_16K)
+               err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_16K, new);
 
        if (!new)
                flush_tlb_kernel_range(PAGE_OFFSET + v, PAGE_OFFSET + top);