2 * PARISC64 Huge TLB page support.
4 * This parisc implementation is heavily based on the SPARC and x86 code.
6 * Copyright (C) 2015 Helge Deller <deller@gmx.de>
11 #include <linux/sched/mm.h>
12 #include <linux/hugetlb.h>
13 #include <linux/pagemap.h>
14 #include <linux/sysctl.h>
17 #include <asm/pgalloc.h>
19 #include <asm/tlbflush.h>
20 #include <asm/cacheflush.h>
21 #include <asm/mmu_context.h>
25 hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
26 unsigned long len
, unsigned long pgoff
, unsigned long flags
)
28 struct hstate
*h
= hstate_file(file
);
30 if (len
& ~huge_page_mask(h
))
35 if (flags
& MAP_FIXED
)
36 if (prepare_hugepage_range(file
, addr
, len
))
40 addr
= ALIGN(addr
, huge_page_size(h
));
42 /* we need to make sure the colouring is OK */
43 return arch_get_unmapped_area(file
, addr
, len
, pgoff
, flags
);
47 pte_t
*huge_pte_alloc(struct mm_struct
*mm
,
48 unsigned long addr
, unsigned long sz
)
55 /* We must align the address, because our caller will run
56 * set_huge_pte_at() on whatever we return, which writes out
57 * all of the sub-ptes for the hugepage range. So we have
58 * to give it the first such sub-pte.
62 pgd
= pgd_offset(mm
, addr
);
63 pud
= pud_alloc(mm
, pgd
, addr
);
65 pmd
= pmd_alloc(mm
, pud
, addr
);
67 pte
= pte_alloc_map(mm
, pmd
, addr
);
72 pte_t
*huge_pte_offset(struct mm_struct
*mm
,
73 unsigned long addr
, unsigned long sz
)
82 pgd
= pgd_offset(mm
, addr
);
83 if (!pgd_none(*pgd
)) {
84 pud
= pud_offset(pgd
, addr
);
85 if (!pud_none(*pud
)) {
86 pmd
= pmd_offset(pud
, addr
);
88 pte
= pte_offset_map(pmd
, addr
);
94 /* Purge data and instruction TLB entries. Must be called holding
95 * the pa_tlb_lock. The TLB purge instructions are slow on SMP
96 * machines since the purge must be broadcast to all CPUs.
98 static inline void purge_tlb_entries_huge(struct mm_struct
*mm
, unsigned long addr
)
102 /* We may use multiple physical huge pages (e.g. 2x1 MB) to emulate
103 * Linux standard huge pages (e.g. 2 MB) */
104 BUILD_BUG_ON(REAL_HPAGE_SHIFT
> HPAGE_SHIFT
);
107 addr
|= _HUGE_PAGE_SIZE_ENCODING_DEFAULT
;
109 for (i
= 0; i
< (1 << (HPAGE_SHIFT
-REAL_HPAGE_SHIFT
)); i
++) {
110 purge_tlb_entries(mm
, addr
);
111 addr
+= (1UL << REAL_HPAGE_SHIFT
);
115 /* __set_huge_pte_at() must be called holding the pa_tlb_lock. */
116 static void __set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
117 pte_t
*ptep
, pte_t entry
)
119 unsigned long addr_start
;
125 for (i
= 0; i
< (1 << HUGETLB_PAGE_ORDER
); i
++) {
126 set_pte(ptep
, entry
);
130 pte_val(entry
) += PAGE_SIZE
;
133 purge_tlb_entries_huge(mm
, addr_start
);
136 void set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
137 pte_t
*ptep
, pte_t entry
)
141 purge_tlb_start(flags
);
142 __set_huge_pte_at(mm
, addr
, ptep
, entry
);
143 purge_tlb_end(flags
);
147 pte_t
huge_ptep_get_and_clear(struct mm_struct
*mm
, unsigned long addr
,
153 purge_tlb_start(flags
);
155 __set_huge_pte_at(mm
, addr
, ptep
, __pte(0));
156 purge_tlb_end(flags
);
162 void huge_ptep_set_wrprotect(struct mm_struct
*mm
,
163 unsigned long addr
, pte_t
*ptep
)
168 purge_tlb_start(flags
);
170 __set_huge_pte_at(mm
, addr
, ptep
, pte_wrprotect(old_pte
));
171 purge_tlb_end(flags
);
174 int huge_ptep_set_access_flags(struct vm_area_struct
*vma
,
175 unsigned long addr
, pte_t
*ptep
,
176 pte_t pte
, int dirty
)
181 purge_tlb_start(flags
);
182 changed
= !pte_same(*ptep
, pte
);
184 __set_huge_pte_at(vma
->vm_mm
, addr
, ptep
, pte
);
186 purge_tlb_end(flags
);
191 int pmd_huge(pmd_t pmd
)
196 int pud_huge(pud_t pud
)