2 * PPC Huge TLB Page Support for Kernel.
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
13 #include <linux/slab.h>
14 #include <linux/hugetlb.h>
15 #include <linux/export.h>
16 #include <linux/of_fdt.h>
17 #include <linux/memblock.h>
18 #include <linux/moduleparam.h>
19 #include <linux/swap.h>
20 #include <linux/swapops.h>
21 #include <linux/kmemleak.h>
22 #include <asm/pgtable.h>
23 #include <asm/pgalloc.h>
25 #include <asm/setup.h>
26 #include <asm/hugetlb.h>
27 #include <asm/pte-walk.h>
29 bool hugetlb_disabled
= false;
31 #define hugepd_none(hpd) (hpd_val(hpd) == 0)
33 #define PTE_T_ORDER (__builtin_ffs(sizeof(pte_t)) - __builtin_ffs(sizeof(void *)))
35 pte_t
*huge_pte_offset(struct mm_struct
*mm
, unsigned long addr
, unsigned long sz
)
38 * Only called for hugetlbfs pages, hence can ignore THP and the
41 return __find_linux_pte(mm
->pgd
, addr
, NULL
, NULL
);
44 static int __hugepte_alloc(struct mm_struct
*mm
, hugepd_t
*hpdp
,
45 unsigned long address
, unsigned int pdshift
,
46 unsigned int pshift
, spinlock_t
*ptl
)
48 struct kmem_cache
*cachep
;
53 if (pshift
>= pdshift
) {
54 cachep
= PGT_CACHE(PTE_T_ORDER
);
55 num_hugepd
= 1 << (pshift
- pdshift
);
56 } else if (IS_ENABLED(CONFIG_PPC_8xx
)) {
57 cachep
= PGT_CACHE(PTE_INDEX_SIZE
);
60 cachep
= PGT_CACHE(pdshift
- pshift
);
65 WARN_ONCE(1, "No page table cache created for hugetlb tables");
69 new = kmem_cache_alloc(cachep
, pgtable_gfp_flags(mm
, GFP_KERNEL
));
71 BUG_ON(pshift
> HUGEPD_SHIFT_MASK
);
72 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK
);
78 * Make sure other cpus find the hugepd set only after a
79 * properly initialized page table is visible to them.
80 * For more details look for comment in __pte_alloc().
86 * We have multiple higher-level entries that point to the same
87 * actual pte location. Fill in each as we go and backtrack on error.
88 * We need all of these so the DTLB pgtable walk code can find the
89 * right higher-level entry without knowing if it's a hugepage or not.
91 for (i
= 0; i
< num_hugepd
; i
++, hpdp
++) {
92 if (unlikely(!hugepd_none(*hpdp
)))
94 hugepd_populate(hpdp
, new, pshift
);
96 /* If we bailed from the for loop early, an error occurred, clean up */
98 for (i
= i
- 1 ; i
>= 0; i
--, hpdp
--)
100 kmem_cache_free(cachep
, new);
102 kmemleak_ignore(new);
109 * At this point we do the placement change only for BOOK3S 64. This would
110 * possibly work on other subarchs.
112 pte_t
*huge_pte_alloc(struct mm_struct
*mm
, unsigned long addr
, unsigned long sz
)
117 hugepd_t
*hpdp
= NULL
;
118 unsigned pshift
= __ffs(sz
);
119 unsigned pdshift
= PGDIR_SHIFT
;
123 pg
= pgd_offset(mm
, addr
);
125 #ifdef CONFIG_PPC_BOOK3S_64
126 if (pshift
== PGDIR_SHIFT
)
129 else if (pshift
> PUD_SHIFT
) {
131 * We need to use hugepd table
133 ptl
= &mm
->page_table_lock
;
134 hpdp
= (hugepd_t
*)pg
;
137 pu
= pud_alloc(mm
, pg
, addr
);
140 if (pshift
== PUD_SHIFT
)
142 else if (pshift
> PMD_SHIFT
) {
143 ptl
= pud_lockptr(mm
, pu
);
144 hpdp
= (hugepd_t
*)pu
;
147 pm
= pmd_alloc(mm
, pu
, addr
);
150 if (pshift
== PMD_SHIFT
)
154 ptl
= pmd_lockptr(mm
, pm
);
155 hpdp
= (hugepd_t
*)pm
;
160 if (pshift
>= PGDIR_SHIFT
) {
161 ptl
= &mm
->page_table_lock
;
162 hpdp
= (hugepd_t
*)pg
;
165 pu
= pud_alloc(mm
, pg
, addr
);
168 if (pshift
>= PUD_SHIFT
) {
169 ptl
= pud_lockptr(mm
, pu
);
170 hpdp
= (hugepd_t
*)pu
;
173 pm
= pmd_alloc(mm
, pu
, addr
);
176 ptl
= pmd_lockptr(mm
, pm
);
177 hpdp
= (hugepd_t
*)pm
;
184 BUG_ON(!hugepd_none(*hpdp
) && !hugepd_ok(*hpdp
));
186 if (hugepd_none(*hpdp
) && __hugepte_alloc(mm
, hpdp
, addr
,
187 pdshift
, pshift
, ptl
))
190 return hugepte_offset(*hpdp
, addr
, pdshift
);
193 #ifdef CONFIG_PPC_BOOK3S_64
195 * Tracks gpages after the device tree is scanned and before the
196 * huge_boot_pages list is ready on pseries.
198 #define MAX_NUMBER_GPAGES 1024
199 __initdata
static u64 gpage_freearray
[MAX_NUMBER_GPAGES
];
200 __initdata
static unsigned nr_gpages
;
203 * Build list of addresses of gigantic pages. This function is used in early
204 * boot before the buddy allocator is setup.
206 void __init
pseries_add_gpage(u64 addr
, u64 page_size
, unsigned long number_of_pages
)
210 while (number_of_pages
> 0) {
211 gpage_freearray
[nr_gpages
] = addr
;
218 int __init
pseries_alloc_bootmem_huge_page(struct hstate
*hstate
)
220 struct huge_bootmem_page
*m
;
223 m
= phys_to_virt(gpage_freearray
[--nr_gpages
]);
224 gpage_freearray
[nr_gpages
] = 0;
225 list_add(&m
->list
, &huge_boot_pages
);
232 int __init
alloc_bootmem_huge_page(struct hstate
*h
)
235 #ifdef CONFIG_PPC_BOOK3S_64
236 if (firmware_has_feature(FW_FEATURE_LPAR
) && !radix_enabled())
237 return pseries_alloc_bootmem_huge_page(h
);
239 return __alloc_bootmem_huge_page(h
);
242 #ifndef CONFIG_PPC_BOOK3S_64
243 #define HUGEPD_FREELIST_SIZE \
244 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
246 struct hugepd_freelist
{
252 static DEFINE_PER_CPU(struct hugepd_freelist
*, hugepd_freelist_cur
);
254 static void hugepd_free_rcu_callback(struct rcu_head
*head
)
256 struct hugepd_freelist
*batch
=
257 container_of(head
, struct hugepd_freelist
, rcu
);
260 for (i
= 0; i
< batch
->index
; i
++)
261 kmem_cache_free(PGT_CACHE(PTE_T_ORDER
), batch
->ptes
[i
]);
263 free_page((unsigned long)batch
);
266 static void hugepd_free(struct mmu_gather
*tlb
, void *hugepte
)
268 struct hugepd_freelist
**batchp
;
270 batchp
= &get_cpu_var(hugepd_freelist_cur
);
272 if (atomic_read(&tlb
->mm
->mm_users
) < 2 ||
273 mm_is_thread_local(tlb
->mm
)) {
274 kmem_cache_free(PGT_CACHE(PTE_T_ORDER
), hugepte
);
275 put_cpu_var(hugepd_freelist_cur
);
279 if (*batchp
== NULL
) {
280 *batchp
= (struct hugepd_freelist
*)__get_free_page(GFP_ATOMIC
);
281 (*batchp
)->index
= 0;
284 (*batchp
)->ptes
[(*batchp
)->index
++] = hugepte
;
285 if ((*batchp
)->index
== HUGEPD_FREELIST_SIZE
) {
286 call_rcu(&(*batchp
)->rcu
, hugepd_free_rcu_callback
);
289 put_cpu_var(hugepd_freelist_cur
);
292 static inline void hugepd_free(struct mmu_gather
*tlb
, void *hugepte
) {}
295 static void free_hugepd_range(struct mmu_gather
*tlb
, hugepd_t
*hpdp
, int pdshift
,
296 unsigned long start
, unsigned long end
,
297 unsigned long floor
, unsigned long ceiling
)
299 pte_t
*hugepte
= hugepd_page(*hpdp
);
302 unsigned long pdmask
= ~((1UL << pdshift
) - 1);
303 unsigned int num_hugepd
= 1;
304 unsigned int shift
= hugepd_shift(*hpdp
);
306 /* Note: On fsl the hpdp may be the first of several */
308 num_hugepd
= 1 << (shift
- pdshift
);
318 if (end
- 1 > ceiling
- 1)
321 for (i
= 0; i
< num_hugepd
; i
++, hpdp
++)
324 if (shift
>= pdshift
)
325 hugepd_free(tlb
, hugepte
);
326 else if (IS_ENABLED(CONFIG_PPC_8xx
))
327 pgtable_free_tlb(tlb
, hugepte
,
328 get_hugepd_cache_index(PTE_INDEX_SIZE
));
330 pgtable_free_tlb(tlb
, hugepte
,
331 get_hugepd_cache_index(pdshift
- shift
));
334 static void hugetlb_free_pmd_range(struct mmu_gather
*tlb
, pud_t
*pud
,
335 unsigned long addr
, unsigned long end
,
336 unsigned long floor
, unsigned long ceiling
)
346 pmd
= pmd_offset(pud
, addr
);
347 next
= pmd_addr_end(addr
, end
);
348 if (!is_hugepd(__hugepd(pmd_val(*pmd
)))) {
350 * if it is not hugepd pointer, we should already find
353 WARN_ON(!pmd_none_or_clear_bad(pmd
));
357 * Increment next by the size of the huge mapping since
358 * there may be more than one entry at this level for a
359 * single hugepage, but all of them point to
360 * the same kmem cache that holds the hugepte.
362 more
= addr
+ (1 << hugepd_shift(*(hugepd_t
*)pmd
));
366 free_hugepd_range(tlb
, (hugepd_t
*)pmd
, PMD_SHIFT
,
367 addr
, next
, floor
, ceiling
);
368 } while (addr
= next
, addr
!= end
);
378 if (end
- 1 > ceiling
- 1)
381 pmd
= pmd_offset(pud
, start
);
383 pmd_free_tlb(tlb
, pmd
, start
);
384 mm_dec_nr_pmds(tlb
->mm
);
387 static void hugetlb_free_pud_range(struct mmu_gather
*tlb
, pgd_t
*pgd
,
388 unsigned long addr
, unsigned long end
,
389 unsigned long floor
, unsigned long ceiling
)
397 pud
= pud_offset(pgd
, addr
);
398 next
= pud_addr_end(addr
, end
);
399 if (!is_hugepd(__hugepd(pud_val(*pud
)))) {
400 if (pud_none_or_clear_bad(pud
))
402 hugetlb_free_pmd_range(tlb
, pud
, addr
, next
, floor
,
407 * Increment next by the size of the huge mapping since
408 * there may be more than one entry at this level for a
409 * single hugepage, but all of them point to
410 * the same kmem cache that holds the hugepte.
412 more
= addr
+ (1 << hugepd_shift(*(hugepd_t
*)pud
));
416 free_hugepd_range(tlb
, (hugepd_t
*)pud
, PUD_SHIFT
,
417 addr
, next
, floor
, ceiling
);
419 } while (addr
= next
, addr
!= end
);
425 ceiling
&= PGDIR_MASK
;
429 if (end
- 1 > ceiling
- 1)
432 pud
= pud_offset(pgd
, start
);
434 pud_free_tlb(tlb
, pud
, start
);
435 mm_dec_nr_puds(tlb
->mm
);
439 * This function frees user-level page tables of a process.
441 void hugetlb_free_pgd_range(struct mmu_gather
*tlb
,
442 unsigned long addr
, unsigned long end
,
443 unsigned long floor
, unsigned long ceiling
)
449 * Because there are a number of different possible pagetable
450 * layouts for hugepage ranges, we limit knowledge of how
451 * things should be laid out to the allocation path
452 * (huge_pte_alloc(), above). Everything else works out the
453 * structure as it goes from information in the hugepd
454 * pointers. That means that we can't here use the
455 * optimization used in the normal page free_pgd_range(), of
456 * checking whether we're actually covering a large enough
457 * range to have to do anything at the top level of the walk
458 * instead of at the bottom.
460 * To make sense of this, you should probably go read the big
461 * block comment at the top of the normal free_pgd_range(),
466 next
= pgd_addr_end(addr
, end
);
467 pgd
= pgd_offset(tlb
->mm
, addr
);
468 if (!is_hugepd(__hugepd(pgd_val(*pgd
)))) {
469 if (pgd_none_or_clear_bad(pgd
))
471 hugetlb_free_pud_range(tlb
, pgd
, addr
, next
, floor
, ceiling
);
475 * Increment next by the size of the huge mapping since
476 * there may be more than one entry at the pgd level
477 * for a single hugepage, but all of them point to the
478 * same kmem cache that holds the hugepte.
480 more
= addr
+ (1 << hugepd_shift(*(hugepd_t
*)pgd
));
484 free_hugepd_range(tlb
, (hugepd_t
*)pgd
, PGDIR_SHIFT
,
485 addr
, next
, floor
, ceiling
);
487 } while (addr
= next
, addr
!= end
);
490 struct page
*follow_huge_pd(struct vm_area_struct
*vma
,
491 unsigned long address
, hugepd_t hpd
,
492 int flags
, int pdshift
)
496 struct page
*page
= NULL
;
498 int shift
= hugepd_shift(hpd
);
499 struct mm_struct
*mm
= vma
->vm_mm
;
503 * hugepage directory entries are protected by mm->page_table_lock
504 * Use this instead of huge_pte_lockptr
506 ptl
= &mm
->page_table_lock
;
509 ptep
= hugepte_offset(hpd
, address
, pdshift
);
510 if (pte_present(*ptep
)) {
511 mask
= (1UL << shift
) - 1;
512 page
= pte_page(*ptep
);
513 page
+= ((address
& mask
) >> PAGE_SHIFT
);
514 if (flags
& FOLL_GET
)
517 if (is_hugetlb_entry_migration(*ptep
)) {
519 __migration_entry_wait(mm
, ptep
, ptl
);
527 #ifdef CONFIG_PPC_MM_SLICES
528 unsigned long hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
529 unsigned long len
, unsigned long pgoff
,
532 struct hstate
*hstate
= hstate_file(file
);
533 int mmu_psize
= shift_to_mmu_psize(huge_page_shift(hstate
));
535 #ifdef CONFIG_PPC_RADIX_MMU
537 return radix__hugetlb_get_unmapped_area(file
, addr
, len
,
540 return slice_get_unmapped_area(addr
, len
, flags
, mmu_psize
, 1);
544 unsigned long vma_mmu_pagesize(struct vm_area_struct
*vma
)
546 /* With radix we don't use slice, so derive it from vma*/
547 if (IS_ENABLED(CONFIG_PPC_MM_SLICES
) && !radix_enabled()) {
548 unsigned int psize
= get_slice_psize(vma
->vm_mm
, vma
->vm_start
);
550 return 1UL << mmu_psize_to_shift(psize
);
552 return vma_kernel_pagesize(vma
);
555 static int __init
add_huge_page_size(unsigned long long size
)
557 int shift
= __ffs(size
);
560 /* Check that it is a page size supported by the hardware and
561 * that it fits within pagetable and slice limits. */
562 if (size
<= PAGE_SIZE
|| !is_power_of_2(size
))
565 mmu_psize
= check_and_get_huge_psize(shift
);
569 BUG_ON(mmu_psize_defs
[mmu_psize
].shift
!= shift
);
571 /* Return if huge page size has already been setup */
572 if (size_to_hstate(size
))
575 hugetlb_add_hstate(shift
- PAGE_SHIFT
);
580 static int __init
hugepage_setup_sz(char *str
)
582 unsigned long long size
;
584 size
= memparse(str
, &str
);
586 if (add_huge_page_size(size
) != 0) {
588 pr_err("Invalid huge page size specified(%llu)\n", size
);
593 __setup("hugepagesz=", hugepage_setup_sz
);
595 static int __init
hugetlbpage_init(void)
597 bool configured
= false;
600 if (hugetlb_disabled
) {
601 pr_info("HugeTLB support is disabled!\n");
605 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64
) && !radix_enabled() &&
606 !mmu_has_feature(MMU_FTR_16M_PAGE
))
609 for (psize
= 0; psize
< MMU_PAGE_COUNT
; ++psize
) {
613 if (!mmu_psize_defs
[psize
].shift
)
616 shift
= mmu_psize_to_shift(psize
);
618 #ifdef CONFIG_PPC_BOOK3S_64
619 if (shift
> PGDIR_SHIFT
)
621 else if (shift
> PUD_SHIFT
)
622 pdshift
= PGDIR_SHIFT
;
623 else if (shift
> PMD_SHIFT
)
628 if (shift
< PUD_SHIFT
)
630 else if (shift
< PGDIR_SHIFT
)
633 pdshift
= PGDIR_SHIFT
;
636 if (add_huge_page_size(1ULL << shift
) < 0)
639 * if we have pdshift and shift value same, we don't
640 * use pgt cache for hugepd.
642 if (pdshift
> shift
&& IS_ENABLED(CONFIG_PPC_8xx
))
643 pgtable_cache_add(PTE_INDEX_SIZE
);
644 else if (pdshift
> shift
)
645 pgtable_cache_add(pdshift
- shift
);
646 else if (IS_ENABLED(CONFIG_PPC_FSL_BOOK3E
) || IS_ENABLED(CONFIG_PPC_8xx
))
647 pgtable_cache_add(PTE_T_ORDER
);
653 if (IS_ENABLED(CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
))
654 hugetlbpage_init_default();
656 pr_info("Failed to initialize. Disabling HugeTLB");
661 arch_initcall(hugetlbpage_init
);
663 void flush_dcache_icache_hugepage(struct page
*page
)
668 BUG_ON(!PageCompound(page
));
670 for (i
= 0; i
< compound_nr(page
); i
++) {
671 if (!PageHighMem(page
)) {
672 __flush_dcache_icache(page_address(page
+i
));
674 start
= kmap_atomic(page
+i
);
675 __flush_dcache_icache(start
);
676 kunmap_atomic(start
);