1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_HUGETLB_H
3 #define _LINUX_HUGETLB_H
6 #include <linux/mm_types.h>
7 #include <linux/mmdebug.h>
9 #include <linux/hugetlb_inline.h>
10 #include <linux/cgroup.h>
11 #include <linux/page_ref.h>
12 #include <linux/list.h>
13 #include <linux/kref.h>
14 #include <linux/pgtable.h>
15 #include <linux/gfp.h>
16 #include <linux/userfaultfd_k.h>
23 #ifndef CONFIG_ARCH_HAS_HUGEPD
24 typedef struct { unsigned long pd
; } hugepd_t
;
25 #define is_hugepd(hugepd) (0)
26 #define __hugepd(x) ((hugepd_t) { (x) })
29 void free_huge_folio(struct folio
*folio
);
31 #ifdef CONFIG_HUGETLB_PAGE
33 #include <linux/mempolicy.h>
34 #include <linux/shm.h>
35 #include <asm/tlbflush.h>
38 * For HugeTLB page, there are more metadata to save in the struct page. But
39 * the head struct page cannot meet our needs, so we have to abuse other tail
40 * struct page to store the metadata.
42 #define __NR_USED_SUBPAGE 3
44 struct hugepage_subpool
{
47 long max_hpages
; /* Maximum huge pages or -1 if no maximum. */
48 long used_hpages
; /* Used count against maximum, includes */
49 /* both allocated and reserved pages. */
50 struct hstate
*hstate
;
51 long min_hpages
; /* Minimum huge pages or -1 if no minimum. */
52 long rsv_hpages
; /* Pages reserved against global pool to */
53 /* satisfy minimum size. */
59 struct list_head regions
;
60 long adds_in_progress
;
61 struct list_head region_cache
;
62 long region_cache_count
;
63 struct rw_semaphore rw_sema
;
64 #ifdef CONFIG_CGROUP_HUGETLB
66 * On private mappings, the counter to uncharge reservations is stored
67 * here. If these fields are 0, then either the mapping is shared, or
68 * cgroup accounting is disabled for this resv_map.
70 struct page_counter
*reservation_counter
;
71 unsigned long pages_per_hpage
;
72 struct cgroup_subsys_state
*css
;
77 * Region tracking -- allows tracking of reservations and instantiated pages
78 * across the pages in a mapping.
80 * The region data structures are embedded into a resv_map and protected
81 * by a resv_map's lock. The set of regions within the resv_map represent
82 * reservations for huge pages, or huge pages that have already been
83 * instantiated within the map. The from and to elements are huge page
84 * indices into the associated mapping. from indicates the starting index
85 * of the region. to represents the first index past the end of the region.
87 * For example, a file region structure with from == 0 and to == 4 represents
88 * four huge pages in a mapping. It is important to note that the to element
89 * represents the first element past the end of the region. This is used in
90 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
92 * Interval notation of the form [from, to) will be used to indicate that
93 * the endpoint from is inclusive and to is exclusive.
96 struct list_head link
;
99 #ifdef CONFIG_CGROUP_HUGETLB
101 * On shared mappings, each reserved region appears as a struct
102 * file_region in resv_map. These fields hold the info needed to
103 * uncharge each reservation.
105 struct page_counter
*reservation_counter
;
106 struct cgroup_subsys_state
*css
;
110 struct hugetlb_vma_lock
{
112 struct rw_semaphore rw_sema
;
113 struct vm_area_struct
*vma
;
116 extern struct resv_map
*resv_map_alloc(void);
117 void resv_map_release(struct kref
*ref
);
119 extern spinlock_t hugetlb_lock
;
120 extern int hugetlb_max_hstate __read_mostly
;
121 #define for_each_hstate(h) \
122 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
124 struct hugepage_subpool
*hugepage_new_subpool(struct hstate
*h
, long max_hpages
,
126 void hugepage_put_subpool(struct hugepage_subpool
*spool
);
128 void hugetlb_dup_vma_private(struct vm_area_struct
*vma
);
129 void clear_vma_resv_huge_pages(struct vm_area_struct
*vma
);
130 int move_hugetlb_page_tables(struct vm_area_struct
*vma
,
131 struct vm_area_struct
*new_vma
,
132 unsigned long old_addr
, unsigned long new_addr
,
134 int copy_hugetlb_page_range(struct mm_struct
*, struct mm_struct
*,
135 struct vm_area_struct
*, struct vm_area_struct
*);
136 struct page
*hugetlb_follow_page_mask(struct vm_area_struct
*vma
,
137 unsigned long address
, unsigned int flags
,
138 unsigned int *page_mask
);
139 void unmap_hugepage_range(struct vm_area_struct
*,
140 unsigned long, unsigned long, struct page
*,
142 void __unmap_hugepage_range(struct mmu_gather
*tlb
,
143 struct vm_area_struct
*vma
,
144 unsigned long start
, unsigned long end
,
145 struct page
*ref_page
, zap_flags_t zap_flags
);
146 void hugetlb_report_meminfo(struct seq_file
*);
147 int hugetlb_report_node_meminfo(char *buf
, int len
, int nid
);
148 void hugetlb_show_meminfo_node(int nid
);
149 unsigned long hugetlb_total_pages(void);
150 vm_fault_t
hugetlb_fault(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
151 unsigned long address
, unsigned int flags
);
152 #ifdef CONFIG_USERFAULTFD
153 int hugetlb_mfill_atomic_pte(pte_t
*dst_pte
,
154 struct vm_area_struct
*dst_vma
,
155 unsigned long dst_addr
,
156 unsigned long src_addr
,
158 struct folio
**foliop
);
159 #endif /* CONFIG_USERFAULTFD */
160 bool hugetlb_reserve_pages(struct inode
*inode
, long from
, long to
,
161 struct vm_area_struct
*vma
,
162 vm_flags_t vm_flags
);
163 long hugetlb_unreserve_pages(struct inode
*inode
, long start
, long end
,
165 bool isolate_hugetlb(struct folio
*folio
, struct list_head
*list
);
166 int get_hwpoison_hugetlb_folio(struct folio
*folio
, bool *hugetlb
, bool unpoison
);
167 int get_huge_page_for_hwpoison(unsigned long pfn
, int flags
,
168 bool *migratable_cleared
);
169 void folio_putback_active_hugetlb(struct folio
*folio
);
170 void move_hugetlb_state(struct folio
*old_folio
, struct folio
*new_folio
, int reason
);
171 void hugetlb_fix_reserve_counts(struct inode
*inode
);
172 extern struct mutex
*hugetlb_fault_mutex_table
;
173 u32
hugetlb_fault_mutex_hash(struct address_space
*mapping
, pgoff_t idx
);
175 pte_t
*huge_pmd_share(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
176 unsigned long addr
, pud_t
*pud
);
178 struct address_space
*hugetlb_page_mapping_lock_write(struct page
*hpage
);
180 extern int sysctl_hugetlb_shm_group
;
181 extern struct list_head huge_boot_pages
;
185 #ifndef CONFIG_HIGHPTE
187 * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures
188 * which may go down to the lowest PTE level in their huge_pte_offset() and
189 * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap().
191 static inline pte_t
*pte_offset_huge(pmd_t
*pmd
, unsigned long address
)
193 return pte_offset_kernel(pmd
, address
);
195 static inline pte_t
*pte_alloc_huge(struct mm_struct
*mm
, pmd_t
*pmd
,
196 unsigned long address
)
198 return pte_alloc(mm
, pmd
) ? NULL
: pte_offset_huge(pmd
, address
);
202 pte_t
*huge_pte_alloc(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
203 unsigned long addr
, unsigned long sz
);
205 * huge_pte_offset(): Walk the hugetlb pgtable until the last level PTE.
206 * Returns the pte_t* if found, or NULL if the address is not mapped.
208 * IMPORTANT: we should normally not directly call this function, instead
209 * this is only a common interface to implement arch-specific
210 * walker. Please use hugetlb_walk() instead, because that will attempt to
211 * verify the locking for you.
213 * Since this function will walk all the pgtable pages (including not only
214 * high-level pgtable page, but also PUD entry that can be unshared
215 * concurrently for VM_SHARED), the caller of this function should be
216 * responsible of its thread safety. One can follow this rule:
218 * (1) For private mappings: pmd unsharing is not possible, so holding the
219 * mmap_lock for either read or write is sufficient. Most callers
220 * already hold the mmap_lock, so normally, no special action is
223 * (2) For shared mappings: pmd unsharing is possible (so the PUD-ranged
224 * pgtable page can go away from under us! It can be done by a pmd
225 * unshare with a follow up munmap() on the other process), then we
228 * (2.1) hugetlb vma lock read or write held, to make sure pmd unshare
229 * won't happen upon the range (it also makes sure the pte_t we
230 * read is the right and stable one), or,
232 * (2.2) hugetlb mapping i_mmap_rwsem lock held read or write, to make
233 * sure even if unshare happened the racy unmap() will wait until
234 * i_mmap_rwsem is released.
236 * Option (2.1) is the safest, which guarantees pte stability from pmd
237 * sharing pov, until the vma lock released. Option (2.2) doesn't protect
238 * a concurrent pmd unshare, but it makes sure the pgtable page is safe to
241 pte_t
*huge_pte_offset(struct mm_struct
*mm
,
242 unsigned long addr
, unsigned long sz
);
243 unsigned long hugetlb_mask_last_page(struct hstate
*h
);
244 int huge_pmd_unshare(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
245 unsigned long addr
, pte_t
*ptep
);
246 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct
*vma
,
247 unsigned long *start
, unsigned long *end
);
249 extern void __hugetlb_zap_begin(struct vm_area_struct
*vma
,
250 unsigned long *begin
, unsigned long *end
);
251 extern void __hugetlb_zap_end(struct vm_area_struct
*vma
,
252 struct zap_details
*details
);
254 static inline void hugetlb_zap_begin(struct vm_area_struct
*vma
,
255 unsigned long *start
, unsigned long *end
)
257 if (is_vm_hugetlb_page(vma
))
258 __hugetlb_zap_begin(vma
, start
, end
);
261 static inline void hugetlb_zap_end(struct vm_area_struct
*vma
,
262 struct zap_details
*details
)
264 if (is_vm_hugetlb_page(vma
))
265 __hugetlb_zap_end(vma
, details
);
268 void hugetlb_vma_lock_read(struct vm_area_struct
*vma
);
269 void hugetlb_vma_unlock_read(struct vm_area_struct
*vma
);
270 void hugetlb_vma_lock_write(struct vm_area_struct
*vma
);
271 void hugetlb_vma_unlock_write(struct vm_area_struct
*vma
);
272 int hugetlb_vma_trylock_write(struct vm_area_struct
*vma
);
273 void hugetlb_vma_assert_locked(struct vm_area_struct
*vma
);
274 void hugetlb_vma_lock_release(struct kref
*kref
);
276 int pmd_huge(pmd_t pmd
);
277 int pud_huge(pud_t pud
);
278 long hugetlb_change_protection(struct vm_area_struct
*vma
,
279 unsigned long address
, unsigned long end
, pgprot_t newprot
,
280 unsigned long cp_flags
);
282 bool is_hugetlb_entry_migration(pte_t pte
);
283 void hugetlb_unshare_all_pmds(struct vm_area_struct
*vma
);
285 #else /* !CONFIG_HUGETLB_PAGE */
287 static inline void hugetlb_dup_vma_private(struct vm_area_struct
*vma
)
291 static inline void clear_vma_resv_huge_pages(struct vm_area_struct
*vma
)
295 static inline unsigned long hugetlb_total_pages(void)
300 static inline struct address_space
*hugetlb_page_mapping_lock_write(
306 static inline int huge_pmd_unshare(struct mm_struct
*mm
,
307 struct vm_area_struct
*vma
,
308 unsigned long addr
, pte_t
*ptep
)
313 static inline void adjust_range_if_pmd_sharing_possible(
314 struct vm_area_struct
*vma
,
315 unsigned long *start
, unsigned long *end
)
319 static inline void hugetlb_zap_begin(
320 struct vm_area_struct
*vma
,
321 unsigned long *start
, unsigned long *end
)
325 static inline void hugetlb_zap_end(
326 struct vm_area_struct
*vma
,
327 struct zap_details
*details
)
331 static inline struct page
*hugetlb_follow_page_mask(
332 struct vm_area_struct
*vma
, unsigned long address
, unsigned int flags
,
333 unsigned int *page_mask
)
335 BUILD_BUG(); /* should never be compiled in if !CONFIG_HUGETLB_PAGE*/
338 static inline int copy_hugetlb_page_range(struct mm_struct
*dst
,
339 struct mm_struct
*src
,
340 struct vm_area_struct
*dst_vma
,
341 struct vm_area_struct
*src_vma
)
347 static inline int move_hugetlb_page_tables(struct vm_area_struct
*vma
,
348 struct vm_area_struct
*new_vma
,
349 unsigned long old_addr
,
350 unsigned long new_addr
,
357 static inline void hugetlb_report_meminfo(struct seq_file
*m
)
361 static inline int hugetlb_report_node_meminfo(char *buf
, int len
, int nid
)
366 static inline void hugetlb_show_meminfo_node(int nid
)
370 static inline int prepare_hugepage_range(struct file
*file
,
371 unsigned long addr
, unsigned long len
)
376 static inline void hugetlb_vma_lock_read(struct vm_area_struct
*vma
)
380 static inline void hugetlb_vma_unlock_read(struct vm_area_struct
*vma
)
384 static inline void hugetlb_vma_lock_write(struct vm_area_struct
*vma
)
388 static inline void hugetlb_vma_unlock_write(struct vm_area_struct
*vma
)
392 static inline int hugetlb_vma_trylock_write(struct vm_area_struct
*vma
)
397 static inline void hugetlb_vma_assert_locked(struct vm_area_struct
*vma
)
401 static inline int pmd_huge(pmd_t pmd
)
406 static inline int pud_huge(pud_t pud
)
411 static inline int is_hugepage_only_range(struct mm_struct
*mm
,
412 unsigned long addr
, unsigned long len
)
417 static inline void hugetlb_free_pgd_range(struct mmu_gather
*tlb
,
418 unsigned long addr
, unsigned long end
,
419 unsigned long floor
, unsigned long ceiling
)
424 #ifdef CONFIG_USERFAULTFD
425 static inline int hugetlb_mfill_atomic_pte(pte_t
*dst_pte
,
426 struct vm_area_struct
*dst_vma
,
427 unsigned long dst_addr
,
428 unsigned long src_addr
,
430 struct folio
**foliop
)
435 #endif /* CONFIG_USERFAULTFD */
437 static inline pte_t
*huge_pte_offset(struct mm_struct
*mm
, unsigned long addr
,
443 static inline bool isolate_hugetlb(struct folio
*folio
, struct list_head
*list
)
448 static inline int get_hwpoison_hugetlb_folio(struct folio
*folio
, bool *hugetlb
, bool unpoison
)
453 static inline int get_huge_page_for_hwpoison(unsigned long pfn
, int flags
,
454 bool *migratable_cleared
)
459 static inline void folio_putback_active_hugetlb(struct folio
*folio
)
463 static inline void move_hugetlb_state(struct folio
*old_folio
,
464 struct folio
*new_folio
, int reason
)
468 static inline long hugetlb_change_protection(
469 struct vm_area_struct
*vma
, unsigned long address
,
470 unsigned long end
, pgprot_t newprot
,
471 unsigned long cp_flags
)
476 static inline void __unmap_hugepage_range(struct mmu_gather
*tlb
,
477 struct vm_area_struct
*vma
, unsigned long start
,
478 unsigned long end
, struct page
*ref_page
,
479 zap_flags_t zap_flags
)
484 static inline vm_fault_t
hugetlb_fault(struct mm_struct
*mm
,
485 struct vm_area_struct
*vma
, unsigned long address
,
492 static inline void hugetlb_unshare_all_pmds(struct vm_area_struct
*vma
) { }
494 #endif /* !CONFIG_HUGETLB_PAGE */
496 * hugepages at page global directory. If arch support
497 * hugepages at pgd level, they need to define this.
500 #define pgd_huge(x) 0
503 #define p4d_huge(x) 0
507 static inline int pgd_write(pgd_t pgd
)
514 #define HUGETLB_ANON_FILE "anon_hugepage"
518 * The file will be used as an shm file so shmfs accounting rules
521 HUGETLB_SHMFS_INODE
= 1,
523 * The file is being created on the internal vfs mount and shmfs
524 * accounting rules do not apply
526 HUGETLB_ANONHUGE_INODE
= 2,
529 #ifdef CONFIG_HUGETLBFS
530 struct hugetlbfs_sb_info
{
531 long max_inodes
; /* inodes allowed */
532 long free_inodes
; /* inodes free */
533 spinlock_t stat_lock
;
534 struct hstate
*hstate
;
535 struct hugepage_subpool
*spool
;
541 static inline struct hugetlbfs_sb_info
*HUGETLBFS_SB(struct super_block
*sb
)
543 return sb
->s_fs_info
;
546 struct hugetlbfs_inode_info
{
547 struct shared_policy policy
;
548 struct inode vfs_inode
;
552 static inline struct hugetlbfs_inode_info
*HUGETLBFS_I(struct inode
*inode
)
554 return container_of(inode
, struct hugetlbfs_inode_info
, vfs_inode
);
557 extern const struct file_operations hugetlbfs_file_operations
;
558 extern const struct vm_operations_struct hugetlb_vm_ops
;
559 struct file
*hugetlb_file_setup(const char *name
, size_t size
, vm_flags_t acct
,
560 int creat_flags
, int page_size_log
);
562 static inline bool is_file_hugepages(struct file
*file
)
564 if (file
->f_op
== &hugetlbfs_file_operations
)
567 return is_file_shm_hugepages(file
);
570 static inline struct hstate
*hstate_inode(struct inode
*i
)
572 return HUGETLBFS_SB(i
->i_sb
)->hstate
;
574 #else /* !CONFIG_HUGETLBFS */
576 #define is_file_hugepages(file) false
577 static inline struct file
*
578 hugetlb_file_setup(const char *name
, size_t size
, vm_flags_t acctflag
,
579 int creat_flags
, int page_size_log
)
581 return ERR_PTR(-ENOSYS
);
584 static inline struct hstate
*hstate_inode(struct inode
*i
)
588 #endif /* !CONFIG_HUGETLBFS */
590 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
591 unsigned long hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
592 unsigned long len
, unsigned long pgoff
,
593 unsigned long flags
);
594 #endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
597 generic_hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
598 unsigned long len
, unsigned long pgoff
,
599 unsigned long flags
);
602 * huegtlb page specific state flags. These flags are located in page.private
603 * of the hugetlb head page. Functions created via the below macros should be
604 * used to manipulate these flags.
606 * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at
607 * allocation time. Cleared when page is fully instantiated. Free
608 * routine checks flag to restore a reservation on error paths.
609 * Synchronization: Examined or modified by code that knows it has
610 * the only reference to page. i.e. After allocation but before use
611 * or when the page is being freed.
612 * HPG_migratable - Set after a newly allocated page is added to the page
613 * cache and/or page tables. Indicates the page is a candidate for
615 * Synchronization: Initially set after new page allocation with no
616 * locking. When examined and modified during migration processing
617 * (isolate, migrate, putback) the hugetlb_lock is held.
618 * HPG_temporary - Set on a page that is temporarily allocated from the buddy
619 * allocator. Typically used for migration target pages when no pages
620 * are available in the pool. The hugetlb free page path will
621 * immediately free pages with this flag set to the buddy allocator.
622 * Synchronization: Can be set after huge page allocation from buddy when
623 * code knows it has only reference. All other examinations and
624 * modifications require hugetlb_lock.
625 * HPG_freed - Set when page is on the free lists.
626 * Synchronization: hugetlb_lock held for examination and modification.
627 * HPG_vmemmap_optimized - Set when the vmemmap pages of the page are freed.
628 * HPG_raw_hwp_unreliable - Set when the hugetlb page has a hwpoison sub-page
629 * that is not tracked by raw_hwp_page list.
631 enum hugetlb_page_flags
{
632 HPG_restore_reserve
= 0,
636 HPG_vmemmap_optimized
,
637 HPG_raw_hwp_unreliable
,
642 * Macros to create test, set and clear function definitions for
643 * hugetlb specific page flags.
645 #ifdef CONFIG_HUGETLB_PAGE
646 #define TESTHPAGEFLAG(uname, flname) \
647 static __always_inline \
648 bool folio_test_hugetlb_##flname(struct folio *folio) \
649 { void *private = &folio->private; \
650 return test_bit(HPG_##flname, private); \
652 static inline int HPage##uname(struct page *page) \
653 { return test_bit(HPG_##flname, &(page->private)); }
655 #define SETHPAGEFLAG(uname, flname) \
656 static __always_inline \
657 void folio_set_hugetlb_##flname(struct folio *folio) \
658 { void *private = &folio->private; \
659 set_bit(HPG_##flname, private); \
661 static inline void SetHPage##uname(struct page *page) \
662 { set_bit(HPG_##flname, &(page->private)); }
664 #define CLEARHPAGEFLAG(uname, flname) \
665 static __always_inline \
666 void folio_clear_hugetlb_##flname(struct folio *folio) \
667 { void *private = &folio->private; \
668 clear_bit(HPG_##flname, private); \
670 static inline void ClearHPage##uname(struct page *page) \
671 { clear_bit(HPG_##flname, &(page->private)); }
673 #define TESTHPAGEFLAG(uname, flname) \
675 folio_test_hugetlb_##flname(struct folio *folio) \
677 static inline int HPage##uname(struct page *page) \
680 #define SETHPAGEFLAG(uname, flname) \
682 folio_set_hugetlb_##flname(struct folio *folio) \
684 static inline void SetHPage##uname(struct page *page) \
687 #define CLEARHPAGEFLAG(uname, flname) \
689 folio_clear_hugetlb_##flname(struct folio *folio) \
691 static inline void ClearHPage##uname(struct page *page) \
695 #define HPAGEFLAG(uname, flname) \
696 TESTHPAGEFLAG(uname, flname) \
697 SETHPAGEFLAG(uname, flname) \
698 CLEARHPAGEFLAG(uname, flname) \
701 * Create functions associated with hugetlb page flags
703 HPAGEFLAG(RestoreReserve
, restore_reserve
)
704 HPAGEFLAG(Migratable
, migratable
)
705 HPAGEFLAG(Temporary
, temporary
)
706 HPAGEFLAG(Freed
, freed
)
707 HPAGEFLAG(VmemmapOptimized
, vmemmap_optimized
)
708 HPAGEFLAG(RawHwpUnreliable
, raw_hwp_unreliable
)
710 #ifdef CONFIG_HUGETLB_PAGE
712 #define HSTATE_NAME_LEN 32
713 /* Defines one hugetlb page size */
715 struct mutex resize_lock
;
716 int next_nid_to_alloc
;
717 int next_nid_to_free
;
719 unsigned int demote_order
;
721 unsigned long max_huge_pages
;
722 unsigned long nr_huge_pages
;
723 unsigned long free_huge_pages
;
724 unsigned long resv_huge_pages
;
725 unsigned long surplus_huge_pages
;
726 unsigned long nr_overcommit_huge_pages
;
727 struct list_head hugepage_activelist
;
728 struct list_head hugepage_freelists
[MAX_NUMNODES
];
729 unsigned int max_huge_pages_node
[MAX_NUMNODES
];
730 unsigned int nr_huge_pages_node
[MAX_NUMNODES
];
731 unsigned int free_huge_pages_node
[MAX_NUMNODES
];
732 unsigned int surplus_huge_pages_node
[MAX_NUMNODES
];
733 #ifdef CONFIG_CGROUP_HUGETLB
734 /* cgroup control files */
735 struct cftype cgroup_files_dfl
[8];
736 struct cftype cgroup_files_legacy
[10];
738 char name
[HSTATE_NAME_LEN
];
741 struct huge_bootmem_page
{
742 struct list_head list
;
743 struct hstate
*hstate
;
746 int isolate_or_dissolve_huge_page(struct page
*page
, struct list_head
*list
);
747 struct folio
*alloc_hugetlb_folio(struct vm_area_struct
*vma
,
748 unsigned long addr
, int avoid_reserve
);
749 struct folio
*alloc_hugetlb_folio_nodemask(struct hstate
*h
, int preferred_nid
,
750 nodemask_t
*nmask
, gfp_t gfp_mask
);
751 struct folio
*alloc_hugetlb_folio_vma(struct hstate
*h
, struct vm_area_struct
*vma
,
752 unsigned long address
);
753 int hugetlb_add_to_page_cache(struct folio
*folio
, struct address_space
*mapping
,
755 void restore_reserve_on_error(struct hstate
*h
, struct vm_area_struct
*vma
,
756 unsigned long address
, struct folio
*folio
);
759 int __init
__alloc_bootmem_huge_page(struct hstate
*h
, int nid
);
760 int __init
alloc_bootmem_huge_page(struct hstate
*h
, int nid
);
761 bool __init
hugetlb_node_alloc_supported(void);
763 void __init
hugetlb_add_hstate(unsigned order
);
764 bool __init
arch_hugetlb_valid_size(unsigned long size
);
765 struct hstate
*size_to_hstate(unsigned long size
);
767 #ifndef HUGE_MAX_HSTATE
768 #define HUGE_MAX_HSTATE 1
771 extern struct hstate hstates
[HUGE_MAX_HSTATE
];
772 extern unsigned int default_hstate_idx
;
774 #define default_hstate (hstates[default_hstate_idx])
776 static inline struct hugepage_subpool
*hugetlb_folio_subpool(struct folio
*folio
)
778 return folio
->_hugetlb_subpool
;
781 static inline void hugetlb_set_folio_subpool(struct folio
*folio
,
782 struct hugepage_subpool
*subpool
)
784 folio
->_hugetlb_subpool
= subpool
;
787 static inline struct hstate
*hstate_file(struct file
*f
)
789 return hstate_inode(file_inode(f
));
792 static inline struct hstate
*hstate_sizelog(int page_size_log
)
795 return &default_hstate
;
797 if (page_size_log
< BITS_PER_LONG
)
798 return size_to_hstate(1UL << page_size_log
);
803 static inline struct hstate
*hstate_vma(struct vm_area_struct
*vma
)
805 return hstate_file(vma
->vm_file
);
808 static inline unsigned long huge_page_size(const struct hstate
*h
)
810 return (unsigned long)PAGE_SIZE
<< h
->order
;
813 extern unsigned long vma_kernel_pagesize(struct vm_area_struct
*vma
);
815 extern unsigned long vma_mmu_pagesize(struct vm_area_struct
*vma
);
817 static inline unsigned long huge_page_mask(struct hstate
*h
)
822 static inline unsigned int huge_page_order(struct hstate
*h
)
827 static inline unsigned huge_page_shift(struct hstate
*h
)
829 return h
->order
+ PAGE_SHIFT
;
832 static inline bool hstate_is_gigantic(struct hstate
*h
)
834 return huge_page_order(h
) > MAX_ORDER
;
837 static inline unsigned int pages_per_huge_page(const struct hstate
*h
)
839 return 1 << h
->order
;
842 static inline unsigned int blocks_per_huge_page(struct hstate
*h
)
844 return huge_page_size(h
) / 512;
847 #include <asm/hugetlb.h>
849 #ifndef is_hugepage_only_range
850 static inline int is_hugepage_only_range(struct mm_struct
*mm
,
851 unsigned long addr
, unsigned long len
)
855 #define is_hugepage_only_range is_hugepage_only_range
858 #ifndef arch_clear_hugepage_flags
859 static inline void arch_clear_hugepage_flags(struct page
*page
) { }
860 #define arch_clear_hugepage_flags arch_clear_hugepage_flags
863 #ifndef arch_make_huge_pte
864 static inline pte_t
arch_make_huge_pte(pte_t entry
, unsigned int shift
,
867 return pte_mkhuge(entry
);
871 static inline struct hstate
*folio_hstate(struct folio
*folio
)
873 VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio
), folio
);
874 return size_to_hstate(folio_size(folio
));
877 static inline unsigned hstate_index_to_shift(unsigned index
)
879 return hstates
[index
].order
+ PAGE_SHIFT
;
882 static inline int hstate_index(struct hstate
*h
)
887 extern int dissolve_free_huge_page(struct page
*page
);
888 extern int dissolve_free_huge_pages(unsigned long start_pfn
,
889 unsigned long end_pfn
);
891 #ifdef CONFIG_MEMORY_FAILURE
892 extern void folio_clear_hugetlb_hwpoison(struct folio
*folio
);
894 static inline void folio_clear_hugetlb_hwpoison(struct folio
*folio
)
899 #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
900 #ifndef arch_hugetlb_migration_supported
901 static inline bool arch_hugetlb_migration_supported(struct hstate
*h
)
903 if ((huge_page_shift(h
) == PMD_SHIFT
) ||
904 (huge_page_shift(h
) == PUD_SHIFT
) ||
905 (huge_page_shift(h
) == PGDIR_SHIFT
))
912 static inline bool arch_hugetlb_migration_supported(struct hstate
*h
)
918 static inline bool hugepage_migration_supported(struct hstate
*h
)
920 return arch_hugetlb_migration_supported(h
);
924 * Movability check is different as compared to migration check.
925 * It determines whether or not a huge page should be placed on
926 * movable zone or not. Movability of any huge page should be
927 * required only if huge page size is supported for migration.
928 * There won't be any reason for the huge page to be movable if
929 * it is not migratable to start with. Also the size of the huge
930 * page should be large enough to be placed under a movable zone
931 * and still feasible enough to be migratable. Just the presence
932 * in movable zone does not make the migration feasible.
934 * So even though large huge page sizes like the gigantic ones
935 * are migratable they should not be movable because its not
936 * feasible to migrate them from movable zone.
938 static inline bool hugepage_movable_supported(struct hstate
*h
)
940 if (!hugepage_migration_supported(h
))
943 if (hstate_is_gigantic(h
))
948 /* Movability of hugepages depends on migration support. */
949 static inline gfp_t
htlb_alloc_mask(struct hstate
*h
)
951 if (hugepage_movable_supported(h
))
952 return GFP_HIGHUSER_MOVABLE
;
957 static inline gfp_t
htlb_modify_alloc_mask(struct hstate
*h
, gfp_t gfp_mask
)
959 gfp_t modified_mask
= htlb_alloc_mask(h
);
961 /* Some callers might want to enforce node */
962 modified_mask
|= (gfp_mask
& __GFP_THISNODE
);
964 modified_mask
|= (gfp_mask
& __GFP_NOWARN
);
966 return modified_mask
;
969 static inline spinlock_t
*huge_pte_lockptr(struct hstate
*h
,
970 struct mm_struct
*mm
, pte_t
*pte
)
972 if (huge_page_size(h
) == PMD_SIZE
)
973 return pmd_lockptr(mm
, (pmd_t
*) pte
);
974 VM_BUG_ON(huge_page_size(h
) == PAGE_SIZE
);
975 return &mm
->page_table_lock
;
978 #ifndef hugepages_supported
980 * Some platform decide whether they support huge pages at boot
981 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
982 * when there is no such support
984 #define hugepages_supported() (HPAGE_SHIFT != 0)
987 void hugetlb_report_usage(struct seq_file
*m
, struct mm_struct
*mm
);
989 static inline void hugetlb_count_init(struct mm_struct
*mm
)
991 atomic_long_set(&mm
->hugetlb_usage
, 0);
994 static inline void hugetlb_count_add(long l
, struct mm_struct
*mm
)
996 atomic_long_add(l
, &mm
->hugetlb_usage
);
999 static inline void hugetlb_count_sub(long l
, struct mm_struct
*mm
)
1001 atomic_long_sub(l
, &mm
->hugetlb_usage
);
1004 #ifndef huge_ptep_modify_prot_start
1005 #define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
1006 static inline pte_t
huge_ptep_modify_prot_start(struct vm_area_struct
*vma
,
1007 unsigned long addr
, pte_t
*ptep
)
1009 return huge_ptep_get_and_clear(vma
->vm_mm
, addr
, ptep
);
1013 #ifndef huge_ptep_modify_prot_commit
1014 #define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
1015 static inline void huge_ptep_modify_prot_commit(struct vm_area_struct
*vma
,
1016 unsigned long addr
, pte_t
*ptep
,
1017 pte_t old_pte
, pte_t pte
)
1019 unsigned long psize
= huge_page_size(hstate_vma(vma
));
1021 set_huge_pte_at(vma
->vm_mm
, addr
, ptep
, pte
, psize
);
1026 void hugetlb_register_node(struct node
*node
);
1027 void hugetlb_unregister_node(struct node
*node
);
1031 * Check if a given raw @page in a hugepage is HWPOISON.
1033 bool is_raw_hwpoison_page_in_hugepage(struct page
*page
);
1035 #else /* CONFIG_HUGETLB_PAGE */
1038 static inline struct hugepage_subpool
*hugetlb_folio_subpool(struct folio
*folio
)
1043 static inline int isolate_or_dissolve_huge_page(struct page
*page
,
1044 struct list_head
*list
)
1049 static inline struct folio
*alloc_hugetlb_folio(struct vm_area_struct
*vma
,
1056 static inline struct folio
*
1057 alloc_hugetlb_folio_nodemask(struct hstate
*h
, int preferred_nid
,
1058 nodemask_t
*nmask
, gfp_t gfp_mask
)
1063 static inline struct folio
*alloc_hugetlb_folio_vma(struct hstate
*h
,
1064 struct vm_area_struct
*vma
,
1065 unsigned long address
)
1070 static inline int __alloc_bootmem_huge_page(struct hstate
*h
)
1075 static inline struct hstate
*hstate_file(struct file
*f
)
1080 static inline struct hstate
*hstate_sizelog(int page_size_log
)
1085 static inline struct hstate
*hstate_vma(struct vm_area_struct
*vma
)
1090 static inline struct hstate
*folio_hstate(struct folio
*folio
)
1095 static inline struct hstate
*size_to_hstate(unsigned long size
)
1100 static inline unsigned long huge_page_size(struct hstate
*h
)
1105 static inline unsigned long huge_page_mask(struct hstate
*h
)
1110 static inline unsigned long vma_kernel_pagesize(struct vm_area_struct
*vma
)
1115 static inline unsigned long vma_mmu_pagesize(struct vm_area_struct
*vma
)
1120 static inline unsigned int huge_page_order(struct hstate
*h
)
1125 static inline unsigned int huge_page_shift(struct hstate
*h
)
1130 static inline bool hstate_is_gigantic(struct hstate
*h
)
1135 static inline unsigned int pages_per_huge_page(struct hstate
*h
)
1140 static inline unsigned hstate_index_to_shift(unsigned index
)
1145 static inline int hstate_index(struct hstate
*h
)
1150 static inline int dissolve_free_huge_page(struct page
*page
)
1155 static inline int dissolve_free_huge_pages(unsigned long start_pfn
,
1156 unsigned long end_pfn
)
1161 static inline bool hugepage_migration_supported(struct hstate
*h
)
1166 static inline bool hugepage_movable_supported(struct hstate
*h
)
1171 static inline gfp_t
htlb_alloc_mask(struct hstate
*h
)
1176 static inline gfp_t
htlb_modify_alloc_mask(struct hstate
*h
, gfp_t gfp_mask
)
1181 static inline spinlock_t
*huge_pte_lockptr(struct hstate
*h
,
1182 struct mm_struct
*mm
, pte_t
*pte
)
1184 return &mm
->page_table_lock
;
1187 static inline void hugetlb_count_init(struct mm_struct
*mm
)
1191 static inline void hugetlb_report_usage(struct seq_file
*f
, struct mm_struct
*m
)
1195 static inline void hugetlb_count_sub(long l
, struct mm_struct
*mm
)
1199 static inline pte_t
huge_ptep_clear_flush(struct vm_area_struct
*vma
,
1200 unsigned long addr
, pte_t
*ptep
)
1203 return ptep_get(ptep
);
1209 static inline void set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
1210 pte_t
*ptep
, pte_t pte
, unsigned long sz
)
1214 static inline void hugetlb_register_node(struct node
*node
)
1218 static inline void hugetlb_unregister_node(struct node
*node
)
1221 #endif /* CONFIG_HUGETLB_PAGE */
1223 static inline spinlock_t
*huge_pte_lock(struct hstate
*h
,
1224 struct mm_struct
*mm
, pte_t
*pte
)
1228 ptl
= huge_pte_lockptr(h
, mm
, pte
);
1233 #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
1234 extern void __init
hugetlb_cma_reserve(int order
);
1236 static inline __init
void hugetlb_cma_reserve(int order
)
1241 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
1242 static inline bool hugetlb_pmd_shared(pte_t
*pte
)
1244 return page_count(virt_to_page(pte
)) > 1;
1247 static inline bool hugetlb_pmd_shared(pte_t
*pte
)
1253 bool want_pmd_share(struct vm_area_struct
*vma
, unsigned long addr
);
1255 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
1257 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
1260 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1263 static inline bool __vma_shareable_lock(struct vm_area_struct
*vma
)
1265 return (vma
->vm_flags
& VM_MAYSHARE
) && vma
->vm_private_data
;
1268 static inline bool __vma_private_lock(struct vm_area_struct
*vma
)
1270 return (!(vma
->vm_flags
& VM_MAYSHARE
)) && vma
->vm_private_data
;
1274 * Safe version of huge_pte_offset() to check the locks. See comments
1275 * above huge_pte_offset().
1277 static inline pte_t
*
1278 hugetlb_walk(struct vm_area_struct
*vma
, unsigned long addr
, unsigned long sz
)
1280 #if defined(CONFIG_HUGETLB_PAGE) && \
1281 defined(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && defined(CONFIG_LOCKDEP)
1282 struct hugetlb_vma_lock
*vma_lock
= vma
->vm_private_data
;
1285 * If pmd sharing possible, locking needed to safely walk the
1286 * hugetlb pgtables. More information can be found at the comment
1287 * above huge_pte_offset() in the same file.
1289 * NOTE: lockdep_is_held() is only defined with CONFIG_LOCKDEP.
1291 if (__vma_shareable_lock(vma
))
1292 WARN_ON_ONCE(!lockdep_is_held(&vma_lock
->rw_sema
) &&
1294 &vma
->vm_file
->f_mapping
->i_mmap_rwsem
));
1296 return huge_pte_offset(vma
->vm_mm
, addr
, sz
);
1299 #endif /* _LINUX_HUGETLB_H */