1 // SPDX-License-Identifier: GPL-2.0-only
3 * mm_init.c - Memory initialisation verification and debugging
5 * Copyright 2008 IBM Corporation, 2008
6 * Author Mel Gorman <mel@csn.ul.ie>
9 #include <linux/kernel.h>
10 #include <linux/init.h>
11 #include <linux/kobject.h>
12 #include <linux/export.h>
13 #include <linux/memory.h>
14 #include <linux/notifier.h>
15 #include <linux/sched.h>
16 #include <linux/mman.h>
17 #include <linux/memblock.h>
18 #include <linux/page-isolation.h>
19 #include <linux/padata.h>
20 #include <linux/nmi.h>
21 #include <linux/buffer_head.h>
22 #include <linux/kmemleak.h>
23 #include <linux/kfence.h>
24 #include <linux/page_ext.h>
25 #include <linux/pti.h>
26 #include <linux/pgtable.h>
27 #include <linux/swap.h>
28 #include <linux/cma.h>
33 #include <asm/setup.h>
35 #ifdef CONFIG_DEBUG_MEMORY_INIT
36 int __meminitdata mminit_loglevel
;
38 /* The zonelists are simply reported, validation is manual. */
39 void __init
mminit_verify_zonelist(void)
43 if (mminit_loglevel
< MMINIT_VERIFY
)
46 for_each_online_node(nid
) {
47 pg_data_t
*pgdat
= NODE_DATA(nid
);
50 struct zonelist
*zonelist
;
51 int i
, listid
, zoneid
;
53 BUILD_BUG_ON(MAX_ZONELISTS
> 2);
54 for (i
= 0; i
< MAX_ZONELISTS
* MAX_NR_ZONES
; i
++) {
56 /* Identify the zone and nodelist */
57 zoneid
= i
% MAX_NR_ZONES
;
58 listid
= i
/ MAX_NR_ZONES
;
59 zonelist
= &pgdat
->node_zonelists
[listid
];
60 zone
= &pgdat
->node_zones
[zoneid
];
61 if (!populated_zone(zone
))
64 /* Print information about the zonelist */
65 printk(KERN_DEBUG
"mminit::zonelist %s %d:%s = ",
66 listid
> 0 ? "thisnode" : "general", nid
,
69 /* Iterate the zonelist */
70 for_each_zone_zonelist(zone
, z
, zonelist
, zoneid
)
71 pr_cont("%d:%s ", zone_to_nid(zone
), zone
->name
);
77 void __init
mminit_verify_pageflags_layout(void)
80 unsigned long or_mask
, add_mask
;
82 shift
= BITS_PER_LONG
;
83 width
= shift
- SECTIONS_WIDTH
- NODES_WIDTH
- ZONES_WIDTH
84 - LAST_CPUPID_SHIFT
- KASAN_TAG_WIDTH
- LRU_GEN_WIDTH
- LRU_REFS_WIDTH
;
85 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_widths",
86 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
95 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_shifts",
96 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
102 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_pgshifts",
103 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",
104 (unsigned long)SECTIONS_PGSHIFT
,
105 (unsigned long)NODES_PGSHIFT
,
106 (unsigned long)ZONES_PGSHIFT
,
107 (unsigned long)LAST_CPUPID_PGSHIFT
,
108 (unsigned long)KASAN_TAG_PGSHIFT
);
109 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_nodezoneid",
110 "Node/Zone ID: %lu -> %lu\n",
111 (unsigned long)(ZONEID_PGOFF
+ ZONEID_SHIFT
),
112 (unsigned long)ZONEID_PGOFF
);
113 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_usage",
114 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",
115 shift
, width
, width
, NR_PAGEFLAGS
, NR_PAGEFLAGS
, 0);
116 #ifdef NODE_NOT_IN_PAGE_FLAGS
117 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_nodeflags",
118 "Node not in page flags");
120 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
121 mminit_dprintk(MMINIT_TRACE
, "pageflags_layout_nodeflags",
122 "Last cpupid not in page flags");
125 if (SECTIONS_WIDTH
) {
126 shift
-= SECTIONS_WIDTH
;
127 BUG_ON(shift
!= SECTIONS_PGSHIFT
);
130 shift
-= NODES_WIDTH
;
131 BUG_ON(shift
!= NODES_PGSHIFT
);
134 shift
-= ZONES_WIDTH
;
135 BUG_ON(shift
!= ZONES_PGSHIFT
);
138 /* Check for bitmask overlaps */
139 or_mask
= (ZONES_MASK
<< ZONES_PGSHIFT
) |
140 (NODES_MASK
<< NODES_PGSHIFT
) |
141 (SECTIONS_MASK
<< SECTIONS_PGSHIFT
);
142 add_mask
= (ZONES_MASK
<< ZONES_PGSHIFT
) +
143 (NODES_MASK
<< NODES_PGSHIFT
) +
144 (SECTIONS_MASK
<< SECTIONS_PGSHIFT
);
145 BUG_ON(or_mask
!= add_mask
);
148 static __init
int set_mminit_loglevel(char *str
)
150 get_option(&str
, &mminit_loglevel
);
153 early_param("mminit_loglevel", set_mminit_loglevel
);
154 #endif /* CONFIG_DEBUG_MEMORY_INIT */
156 struct kobject
*mm_kobj
;
159 s32 vm_committed_as_batch
= 32;
161 void mm_compute_batch(int overcommit_policy
)
164 s32 nr
= num_present_cpus();
165 s32 batch
= max_t(s32
, nr
*2, 32);
166 unsigned long ram_pages
= totalram_pages();
169 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of
170 * (total memory/#cpus), and lift it to 25% for other policies
171 * to easy the possible lock contention for percpu_counter
172 * vm_committed_as, while the max limit is INT_MAX
174 if (overcommit_policy
== OVERCOMMIT_NEVER
)
175 memsized_batch
= min_t(u64
, ram_pages
/nr
/256, INT_MAX
);
177 memsized_batch
= min_t(u64
, ram_pages
/nr
/4, INT_MAX
);
179 vm_committed_as_batch
= max_t(s32
, memsized_batch
, batch
);
182 static int __meminit
mm_compute_batch_notifier(struct notifier_block
*self
,
183 unsigned long action
, void *arg
)
188 mm_compute_batch(sysctl_overcommit_memory
);
196 static int __init
mm_compute_batch_init(void)
198 mm_compute_batch(sysctl_overcommit_memory
);
199 hotplug_memory_notifier(mm_compute_batch_notifier
, MM_COMPUTE_BATCH_PRI
);
203 __initcall(mm_compute_batch_init
);
207 static int __init
mm_sysfs_init(void)
209 mm_kobj
= kobject_create_and_add("mm", kernel_kobj
);
215 postcore_initcall(mm_sysfs_init
);
217 static unsigned long arch_zone_lowest_possible_pfn
[MAX_NR_ZONES
] __initdata
;
218 static unsigned long arch_zone_highest_possible_pfn
[MAX_NR_ZONES
] __initdata
;
219 static unsigned long zone_movable_pfn
[MAX_NUMNODES
] __initdata
;
221 static unsigned long required_kernelcore __initdata
;
222 static unsigned long required_kernelcore_percent __initdata
;
223 static unsigned long required_movablecore __initdata
;
224 static unsigned long required_movablecore_percent __initdata
;
226 static unsigned long nr_kernel_pages __initdata
;
227 static unsigned long nr_all_pages __initdata
;
228 static unsigned long dma_reserve __initdata
;
230 static bool deferred_struct_pages __meminitdata
;
232 static DEFINE_PER_CPU(struct per_cpu_nodestat
, boot_nodestats
);
234 static int __init
cmdline_parse_core(char *p
, unsigned long *core
,
235 unsigned long *percent
)
237 unsigned long long coremem
;
243 /* Value may be a percentage of total memory, otherwise bytes */
244 coremem
= simple_strtoull(p
, &endptr
, 0);
245 if (*endptr
== '%') {
246 /* Paranoid check for percent values greater than 100 */
247 WARN_ON(coremem
> 100);
251 coremem
= memparse(p
, &p
);
252 /* Paranoid check that UL is enough for the coremem value */
253 WARN_ON((coremem
>> PAGE_SHIFT
) > ULONG_MAX
);
255 *core
= coremem
>> PAGE_SHIFT
;
261 bool mirrored_kernelcore __initdata_memblock
;
264 * kernelcore=size sets the amount of memory for use for allocations that
265 * cannot be reclaimed or migrated.
267 static int __init
cmdline_parse_kernelcore(char *p
)
269 /* parse kernelcore=mirror */
270 if (parse_option_str(p
, "mirror")) {
271 mirrored_kernelcore
= true;
275 return cmdline_parse_core(p
, &required_kernelcore
,
276 &required_kernelcore_percent
);
278 early_param("kernelcore", cmdline_parse_kernelcore
);
281 * movablecore=size sets the amount of memory for use for allocations that
282 * can be reclaimed or migrated.
284 static int __init
cmdline_parse_movablecore(char *p
)
286 return cmdline_parse_core(p
, &required_movablecore
,
287 &required_movablecore_percent
);
289 early_param("movablecore", cmdline_parse_movablecore
);
292 * early_calculate_totalpages()
293 * Sum pages in active regions for movable zone.
294 * Populate N_MEMORY for calculating usable_nodes.
296 static unsigned long __init
early_calculate_totalpages(void)
298 unsigned long totalpages
= 0;
299 unsigned long start_pfn
, end_pfn
;
302 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start_pfn
, &end_pfn
, &nid
) {
303 unsigned long pages
= end_pfn
- start_pfn
;
307 node_set_state(nid
, N_MEMORY
);
313 * This finds a zone that can be used for ZONE_MOVABLE pages. The
314 * assumption is made that zones within a node are ordered in monotonic
315 * increasing memory addresses so that the "highest" populated zone is used
317 static void __init
find_usable_zone_for_movable(void)
320 for (zone_index
= MAX_NR_ZONES
- 1; zone_index
>= 0; zone_index
--) {
321 if (zone_index
== ZONE_MOVABLE
)
324 if (arch_zone_highest_possible_pfn
[zone_index
] >
325 arch_zone_lowest_possible_pfn
[zone_index
])
329 VM_BUG_ON(zone_index
== -1);
330 movable_zone
= zone_index
;
334 * Find the PFN the Movable zone begins in each node. Kernel memory
335 * is spread evenly between nodes as long as the nodes have enough
336 * memory. When they don't, some nodes will have more kernelcore than
339 static void __init
find_zone_movable_pfns_for_nodes(void)
342 unsigned long usable_startpfn
;
343 unsigned long kernelcore_node
, kernelcore_remaining
;
344 /* save the state before borrow the nodemask */
345 nodemask_t saved_node_state
= node_states
[N_MEMORY
];
346 unsigned long totalpages
= early_calculate_totalpages();
347 int usable_nodes
= nodes_weight(node_states
[N_MEMORY
]);
348 struct memblock_region
*r
;
350 /* Need to find movable_zone earlier when movable_node is specified. */
351 find_usable_zone_for_movable();
354 * If movable_node is specified, ignore kernelcore and movablecore
357 if (movable_node_is_enabled()) {
358 for_each_mem_region(r
) {
359 if (!memblock_is_hotpluggable(r
))
362 nid
= memblock_get_region_node(r
);
364 usable_startpfn
= PFN_DOWN(r
->base
);
365 zone_movable_pfn
[nid
] = zone_movable_pfn
[nid
] ?
366 min(usable_startpfn
, zone_movable_pfn
[nid
]) :
374 * If kernelcore=mirror is specified, ignore movablecore option
376 if (mirrored_kernelcore
) {
377 bool mem_below_4gb_not_mirrored
= false;
379 if (!memblock_has_mirror()) {
380 pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n");
384 for_each_mem_region(r
) {
385 if (memblock_is_mirror(r
))
388 nid
= memblock_get_region_node(r
);
390 usable_startpfn
= memblock_region_memory_base_pfn(r
);
392 if (usable_startpfn
< PHYS_PFN(SZ_4G
)) {
393 mem_below_4gb_not_mirrored
= true;
397 zone_movable_pfn
[nid
] = zone_movable_pfn
[nid
] ?
398 min(usable_startpfn
, zone_movable_pfn
[nid
]) :
402 if (mem_below_4gb_not_mirrored
)
403 pr_warn("This configuration results in unmirrored kernel memory.\n");
409 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
410 * amount of necessary memory.
412 if (required_kernelcore_percent
)
413 required_kernelcore
= (totalpages
* 100 * required_kernelcore_percent
) /
415 if (required_movablecore_percent
)
416 required_movablecore
= (totalpages
* 100 * required_movablecore_percent
) /
420 * If movablecore= was specified, calculate what size of
421 * kernelcore that corresponds so that memory usable for
422 * any allocation type is evenly spread. If both kernelcore
423 * and movablecore are specified, then the value of kernelcore
424 * will be used for required_kernelcore if it's greater than
425 * what movablecore would have allowed.
427 if (required_movablecore
) {
428 unsigned long corepages
;
431 * Round-up so that ZONE_MOVABLE is at least as large as what
432 * was requested by the user
434 required_movablecore
=
435 roundup(required_movablecore
, MAX_ORDER_NR_PAGES
);
436 required_movablecore
= min(totalpages
, required_movablecore
);
437 corepages
= totalpages
- required_movablecore
;
439 required_kernelcore
= max(required_kernelcore
, corepages
);
443 * If kernelcore was not specified or kernelcore size is larger
444 * than totalpages, there is no ZONE_MOVABLE.
446 if (!required_kernelcore
|| required_kernelcore
>= totalpages
)
449 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
450 usable_startpfn
= arch_zone_lowest_possible_pfn
[movable_zone
];
453 /* Spread kernelcore memory as evenly as possible throughout nodes */
454 kernelcore_node
= required_kernelcore
/ usable_nodes
;
455 for_each_node_state(nid
, N_MEMORY
) {
456 unsigned long start_pfn
, end_pfn
;
459 * Recalculate kernelcore_node if the division per node
460 * now exceeds what is necessary to satisfy the requested
461 * amount of memory for the kernel
463 if (required_kernelcore
< kernelcore_node
)
464 kernelcore_node
= required_kernelcore
/ usable_nodes
;
467 * As the map is walked, we track how much memory is usable
468 * by the kernel using kernelcore_remaining. When it is
469 * 0, the rest of the node is usable by ZONE_MOVABLE
471 kernelcore_remaining
= kernelcore_node
;
473 /* Go through each range of PFNs within this node */
474 for_each_mem_pfn_range(i
, nid
, &start_pfn
, &end_pfn
, NULL
) {
475 unsigned long size_pages
;
477 start_pfn
= max(start_pfn
, zone_movable_pfn
[nid
]);
478 if (start_pfn
>= end_pfn
)
481 /* Account for what is only usable for kernelcore */
482 if (start_pfn
< usable_startpfn
) {
483 unsigned long kernel_pages
;
484 kernel_pages
= min(end_pfn
, usable_startpfn
)
487 kernelcore_remaining
-= min(kernel_pages
,
488 kernelcore_remaining
);
489 required_kernelcore
-= min(kernel_pages
,
490 required_kernelcore
);
492 /* Continue if range is now fully accounted */
493 if (end_pfn
<= usable_startpfn
) {
496 * Push zone_movable_pfn to the end so
497 * that if we have to rebalance
498 * kernelcore across nodes, we will
499 * not double account here
501 zone_movable_pfn
[nid
] = end_pfn
;
504 start_pfn
= usable_startpfn
;
508 * The usable PFN range for ZONE_MOVABLE is from
509 * start_pfn->end_pfn. Calculate size_pages as the
510 * number of pages used as kernelcore
512 size_pages
= end_pfn
- start_pfn
;
513 if (size_pages
> kernelcore_remaining
)
514 size_pages
= kernelcore_remaining
;
515 zone_movable_pfn
[nid
] = start_pfn
+ size_pages
;
518 * Some kernelcore has been met, update counts and
519 * break if the kernelcore for this node has been
522 required_kernelcore
-= min(required_kernelcore
,
524 kernelcore_remaining
-= size_pages
;
525 if (!kernelcore_remaining
)
531 * If there is still required_kernelcore, we do another pass with one
532 * less node in the count. This will push zone_movable_pfn[nid] further
533 * along on the nodes that still have memory until kernelcore is
537 if (usable_nodes
&& required_kernelcore
> usable_nodes
)
541 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
542 for (nid
= 0; nid
< MAX_NUMNODES
; nid
++) {
543 unsigned long start_pfn
, end_pfn
;
545 zone_movable_pfn
[nid
] =
546 roundup(zone_movable_pfn
[nid
], MAX_ORDER_NR_PAGES
);
548 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
549 if (zone_movable_pfn
[nid
] >= end_pfn
)
550 zone_movable_pfn
[nid
] = 0;
554 /* restore the node_state */
555 node_states
[N_MEMORY
] = saved_node_state
;
558 static void __meminit
__init_single_page(struct page
*page
, unsigned long pfn
,
559 unsigned long zone
, int nid
)
561 mm_zero_struct_page(page
);
562 set_page_links(page
, zone
, nid
, pfn
);
563 init_page_count(page
);
564 page_mapcount_reset(page
);
565 page_cpupid_reset_last(page
);
566 page_kasan_tag_reset(page
);
568 INIT_LIST_HEAD(&page
->lru
);
569 #ifdef WANT_PAGE_VIRTUAL
570 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
571 if (!is_highmem_idx(zone
))
572 set_page_address(page
, __va(pfn
<< PAGE_SHIFT
));
578 * During memory init memblocks map pfns to nids. The search is expensive and
579 * this caches recent lookups. The implementation of __early_pfn_to_nid
580 * treats start/end as pfns.
582 struct mminit_pfnnid_cache
{
583 unsigned long last_start
;
584 unsigned long last_end
;
588 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata
;
591 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
593 static int __meminit
__early_pfn_to_nid(unsigned long pfn
,
594 struct mminit_pfnnid_cache
*state
)
596 unsigned long start_pfn
, end_pfn
;
599 if (state
->last_start
<= pfn
&& pfn
< state
->last_end
)
600 return state
->last_nid
;
602 nid
= memblock_search_pfn_nid(pfn
, &start_pfn
, &end_pfn
);
603 if (nid
!= NUMA_NO_NODE
) {
604 state
->last_start
= start_pfn
;
605 state
->last_end
= end_pfn
;
606 state
->last_nid
= nid
;
612 int __meminit
early_pfn_to_nid(unsigned long pfn
)
614 static DEFINE_SPINLOCK(early_pfn_lock
);
617 spin_lock(&early_pfn_lock
);
618 nid
= __early_pfn_to_nid(pfn
, &early_pfnnid_cache
);
620 nid
= first_online_node
;
621 spin_unlock(&early_pfn_lock
);
626 int hashdist
= HASHDIST_DEFAULT
;
628 static int __init
set_hashdist(char *str
)
632 hashdist
= simple_strtoul(str
, &str
, 0);
635 __setup("hashdist=", set_hashdist
);
637 static inline void fixup_hashdist(void)
639 if (num_node_state(N_MEMORY
) == 1)
643 static inline void fixup_hashdist(void) {}
644 #endif /* CONFIG_NUMA */
646 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
647 static inline void pgdat_set_deferred_range(pg_data_t
*pgdat
)
649 pgdat
->first_deferred_pfn
= ULONG_MAX
;
652 /* Returns true if the struct page for the pfn is initialised */
653 static inline bool __meminit
early_page_initialised(unsigned long pfn
, int nid
)
655 if (node_online(nid
) && pfn
>= NODE_DATA(nid
)->first_deferred_pfn
)
662 * Returns true when the remaining initialisation should be deferred until
663 * later in the boot cycle when it can be parallelised.
665 static bool __meminit
666 defer_init(int nid
, unsigned long pfn
, unsigned long end_pfn
)
668 static unsigned long prev_end_pfn
, nr_initialised
;
670 if (early_page_ext_enabled())
673 * prev_end_pfn static that contains the end of previous zone
674 * No need to protect because called very early in boot before smp_init.
676 if (prev_end_pfn
!= end_pfn
) {
677 prev_end_pfn
= end_pfn
;
681 /* Always populate low zones for address-constrained allocations */
682 if (end_pfn
< pgdat_end_pfn(NODE_DATA(nid
)))
685 if (NODE_DATA(nid
)->first_deferred_pfn
!= ULONG_MAX
)
688 * We start only with one section of pages, more pages are added as
689 * needed until the rest of deferred pages are initialized.
692 if ((nr_initialised
> PAGES_PER_SECTION
) &&
693 (pfn
& (PAGES_PER_SECTION
- 1)) == 0) {
694 NODE_DATA(nid
)->first_deferred_pfn
= pfn
;
700 static void __meminit
init_reserved_page(unsigned long pfn
, int nid
)
705 if (early_page_initialised(pfn
, nid
))
708 pgdat
= NODE_DATA(nid
);
710 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
711 struct zone
*zone
= &pgdat
->node_zones
[zid
];
713 if (zone_spans_pfn(zone
, pfn
))
716 __init_single_page(pfn_to_page(pfn
), pfn
, zid
, nid
);
719 static inline void pgdat_set_deferred_range(pg_data_t
*pgdat
) {}
721 static inline bool early_page_initialised(unsigned long pfn
, int nid
)
726 static inline bool defer_init(int nid
, unsigned long pfn
, unsigned long end_pfn
)
731 static inline void init_reserved_page(unsigned long pfn
, int nid
)
734 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
737 * Initialised pages do not have PageReserved set. This function is
738 * called for each range allocated by the bootmem allocator and
739 * marks the pages PageReserved. The remaining valid pages are later
740 * sent to the buddy page allocator.
742 void __meminit
reserve_bootmem_region(phys_addr_t start
,
743 phys_addr_t end
, int nid
)
745 unsigned long start_pfn
= PFN_DOWN(start
);
746 unsigned long end_pfn
= PFN_UP(end
);
748 for (; start_pfn
< end_pfn
; start_pfn
++) {
749 if (pfn_valid(start_pfn
)) {
750 struct page
*page
= pfn_to_page(start_pfn
);
752 init_reserved_page(start_pfn
, nid
);
754 /* Avoid false-positive PageTail() */
755 INIT_LIST_HEAD(&page
->lru
);
758 * no need for atomic set_bit because the struct
759 * page is not visible yet so nobody should
762 __SetPageReserved(page
);
767 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
768 static bool __meminit
769 overlap_memmap_init(unsigned long zone
, unsigned long *pfn
)
771 static struct memblock_region
*r
;
773 if (mirrored_kernelcore
&& zone
== ZONE_MOVABLE
) {
774 if (!r
|| *pfn
>= memblock_region_memory_end_pfn(r
)) {
775 for_each_mem_region(r
) {
776 if (*pfn
< memblock_region_memory_end_pfn(r
))
780 if (*pfn
>= memblock_region_memory_base_pfn(r
) &&
781 memblock_is_mirror(r
)) {
782 *pfn
= memblock_region_memory_end_pfn(r
);
790 * Only struct pages that correspond to ranges defined by memblock.memory
791 * are zeroed and initialized by going through __init_single_page() during
792 * memmap_init_zone_range().
794 * But, there could be struct pages that correspond to holes in
795 * memblock.memory. This can happen because of the following reasons:
796 * - physical memory bank size is not necessarily the exact multiple of the
797 * arbitrary section size
798 * - early reserved memory may not be listed in memblock.memory
799 * - memory layouts defined with memmap= kernel parameter may not align
800 * nicely with memmap sections
802 * Explicitly initialize those struct pages so that:
803 * - PG_Reserved is set
804 * - zone and node links point to zone and node that span the page if the
805 * hole is in the middle of a zone
806 * - zone and node links point to adjacent zone/node if the hole falls on
807 * the zone boundary; the pages in such holes will be prepended to the
808 * zone/node above the hole except for the trailing pages in the last
809 * section that will be appended to the zone/node below.
811 static void __init
init_unavailable_range(unsigned long spfn
,
818 for (pfn
= spfn
; pfn
< epfn
; pfn
++) {
819 if (!pfn_valid(pageblock_start_pfn(pfn
))) {
820 pfn
= pageblock_end_pfn(pfn
) - 1;
823 __init_single_page(pfn_to_page(pfn
), pfn
, zone
, node
);
824 __SetPageReserved(pfn_to_page(pfn
));
829 pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
830 node
, zone_names
[zone
], pgcnt
);
834 * Initially all pages are reserved - free ones are freed
835 * up by memblock_free_all() once the early boot process is
836 * done. Non-atomic initialization, single-pass.
838 * All aligned pageblocks are initialized to the specified migratetype
839 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
840 * zone stats (e.g., nr_isolate_pageblock) are touched.
842 void __meminit
memmap_init_range(unsigned long size
, int nid
, unsigned long zone
,
843 unsigned long start_pfn
, unsigned long zone_end_pfn
,
844 enum meminit_context context
,
845 struct vmem_altmap
*altmap
, int migratetype
)
847 unsigned long pfn
, end_pfn
= start_pfn
+ size
;
850 if (highest_memmap_pfn
< end_pfn
- 1)
851 highest_memmap_pfn
= end_pfn
- 1;
853 #ifdef CONFIG_ZONE_DEVICE
855 * Honor reservation requested by the driver for this ZONE_DEVICE
856 * memory. We limit the total number of pages to initialize to just
857 * those that might contain the memory mapping. We will defer the
858 * ZONE_DEVICE page initialization until after we have released
861 if (zone
== ZONE_DEVICE
) {
865 if (start_pfn
== altmap
->base_pfn
)
866 start_pfn
+= altmap
->reserve
;
867 end_pfn
= altmap
->base_pfn
+ vmem_altmap_offset(altmap
);
871 for (pfn
= start_pfn
; pfn
< end_pfn
; ) {
873 * There can be holes in boot-time mem_map[]s handed to this
874 * function. They do not exist on hotplugged memory.
876 if (context
== MEMINIT_EARLY
) {
877 if (overlap_memmap_init(zone
, &pfn
))
879 if (defer_init(nid
, pfn
, zone_end_pfn
)) {
880 deferred_struct_pages
= true;
885 page
= pfn_to_page(pfn
);
886 __init_single_page(page
, pfn
, zone
, nid
);
887 if (context
== MEMINIT_HOTPLUG
)
888 __SetPageReserved(page
);
891 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
892 * such that unmovable allocations won't be scattered all
893 * over the place during system boot.
895 if (pageblock_aligned(pfn
)) {
896 set_pageblock_migratetype(page
, migratetype
);
903 static void __init
memmap_init_zone_range(struct zone
*zone
,
904 unsigned long start_pfn
,
905 unsigned long end_pfn
,
906 unsigned long *hole_pfn
)
908 unsigned long zone_start_pfn
= zone
->zone_start_pfn
;
909 unsigned long zone_end_pfn
= zone_start_pfn
+ zone
->spanned_pages
;
910 int nid
= zone_to_nid(zone
), zone_id
= zone_idx(zone
);
912 start_pfn
= clamp(start_pfn
, zone_start_pfn
, zone_end_pfn
);
913 end_pfn
= clamp(end_pfn
, zone_start_pfn
, zone_end_pfn
);
915 if (start_pfn
>= end_pfn
)
918 memmap_init_range(end_pfn
- start_pfn
, nid
, zone_id
, start_pfn
,
919 zone_end_pfn
, MEMINIT_EARLY
, NULL
, MIGRATE_MOVABLE
);
921 if (*hole_pfn
< start_pfn
)
922 init_unavailable_range(*hole_pfn
, start_pfn
, zone_id
, nid
);
927 static void __init
memmap_init(void)
929 unsigned long start_pfn
, end_pfn
;
930 unsigned long hole_pfn
= 0;
931 int i
, j
, zone_id
= 0, nid
;
933 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start_pfn
, &end_pfn
, &nid
) {
934 struct pglist_data
*node
= NODE_DATA(nid
);
936 for (j
= 0; j
< MAX_NR_ZONES
; j
++) {
937 struct zone
*zone
= node
->node_zones
+ j
;
939 if (!populated_zone(zone
))
942 memmap_init_zone_range(zone
, start_pfn
, end_pfn
,
948 #ifdef CONFIG_SPARSEMEM
950 * Initialize the memory map for hole in the range [memory_end,
952 * Append the pages in this hole to the highest zone in the last
954 * The call to init_unavailable_range() is outside the ifdef to
955 * silence the compiler warining about zone_id set but not used;
956 * for FLATMEM it is a nop anyway
958 end_pfn
= round_up(end_pfn
, PAGES_PER_SECTION
);
959 if (hole_pfn
< end_pfn
)
961 init_unavailable_range(hole_pfn
, end_pfn
, zone_id
, nid
);
964 #ifdef CONFIG_ZONE_DEVICE
965 static void __ref
__init_zone_device_page(struct page
*page
, unsigned long pfn
,
966 unsigned long zone_idx
, int nid
,
967 struct dev_pagemap
*pgmap
)
970 __init_single_page(page
, pfn
, zone_idx
, nid
);
973 * Mark page reserved as it will need to wait for onlining
974 * phase for it to be fully associated with a zone.
976 * We can use the non-atomic __set_bit operation for setting
977 * the flag as we are still initializing the pages.
979 __SetPageReserved(page
);
982 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
983 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
984 * ever freed or placed on a driver-private list.
987 page
->zone_device_data
= NULL
;
990 * Mark the block movable so that blocks are reserved for
991 * movable at startup. This will force kernel allocations
992 * to reserve their blocks rather than leaking throughout
993 * the address space during boot when many long-lived
994 * kernel allocations are made.
996 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
997 * because this is done early in section_activate()
999 if (pageblock_aligned(pfn
)) {
1000 set_pageblock_migratetype(page
, MIGRATE_MOVABLE
);
1005 * ZONE_DEVICE pages are released directly to the driver page allocator
1006 * which will set the page count to 1 when allocating the page.
1008 if (pgmap
->type
== MEMORY_DEVICE_PRIVATE
||
1009 pgmap
->type
== MEMORY_DEVICE_COHERENT
)
1010 set_page_count(page
, 0);
1014 * With compound page geometry and when struct pages are stored in ram most
1015 * tail pages are reused. Consequently, the amount of unique struct pages to
1016 * initialize is a lot smaller that the total amount of struct pages being
1017 * mapped. This is a paired / mild layering violation with explicit knowledge
1018 * of how the sparse_vmemmap internals handle compound pages in the lack
1019 * of an altmap. See vmemmap_populate_compound_pages().
1021 static inline unsigned long compound_nr_pages(struct vmem_altmap
*altmap
,
1022 struct dev_pagemap
*pgmap
)
1024 if (!vmemmap_can_optimize(altmap
, pgmap
))
1025 return pgmap_vmemmap_nr(pgmap
);
1027 return VMEMMAP_RESERVE_NR
* (PAGE_SIZE
/ sizeof(struct page
));
1030 static void __ref
memmap_init_compound(struct page
*head
,
1031 unsigned long head_pfn
,
1032 unsigned long zone_idx
, int nid
,
1033 struct dev_pagemap
*pgmap
,
1034 unsigned long nr_pages
)
1036 unsigned long pfn
, end_pfn
= head_pfn
+ nr_pages
;
1037 unsigned int order
= pgmap
->vmemmap_shift
;
1039 __SetPageHead(head
);
1040 for (pfn
= head_pfn
+ 1; pfn
< end_pfn
; pfn
++) {
1041 struct page
*page
= pfn_to_page(pfn
);
1043 __init_zone_device_page(page
, pfn
, zone_idx
, nid
, pgmap
);
1044 prep_compound_tail(head
, pfn
- head_pfn
);
1045 set_page_count(page
, 0);
1048 * The first tail page stores important compound page info.
1049 * Call prep_compound_head() after the first tail page has
1050 * been initialized, to not have the data overwritten.
1052 if (pfn
== head_pfn
+ 1)
1053 prep_compound_head(head
, order
);
1057 void __ref
memmap_init_zone_device(struct zone
*zone
,
1058 unsigned long start_pfn
,
1059 unsigned long nr_pages
,
1060 struct dev_pagemap
*pgmap
)
1062 unsigned long pfn
, end_pfn
= start_pfn
+ nr_pages
;
1063 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
1064 struct vmem_altmap
*altmap
= pgmap_altmap(pgmap
);
1065 unsigned int pfns_per_compound
= pgmap_vmemmap_nr(pgmap
);
1066 unsigned long zone_idx
= zone_idx(zone
);
1067 unsigned long start
= jiffies
;
1068 int nid
= pgdat
->node_id
;
1070 if (WARN_ON_ONCE(!pgmap
|| zone_idx
!= ZONE_DEVICE
))
1074 * The call to memmap_init should have already taken care
1075 * of the pages reserved for the memmap, so we can just jump to
1076 * the end of that region and start processing the device pages.
1079 start_pfn
= altmap
->base_pfn
+ vmem_altmap_offset(altmap
);
1080 nr_pages
= end_pfn
- start_pfn
;
1083 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pfns_per_compound
) {
1084 struct page
*page
= pfn_to_page(pfn
);
1086 __init_zone_device_page(page
, pfn
, zone_idx
, nid
, pgmap
);
1088 if (pfns_per_compound
== 1)
1091 memmap_init_compound(page
, pfn
, zone_idx
, nid
, pgmap
,
1092 compound_nr_pages(altmap
, pgmap
));
1095 pr_debug("%s initialised %lu pages in %ums\n", __func__
,
1096 nr_pages
, jiffies_to_msecs(jiffies
- start
));
1101 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
1102 * because it is sized independent of architecture. Unlike the other zones,
1103 * the starting point for ZONE_MOVABLE is not fixed. It may be different
1104 * in each node depending on the size of each node and how evenly kernelcore
1105 * is distributed. This helper function adjusts the zone ranges
1106 * provided by the architecture for a given node by using the end of the
1107 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
1108 * zones within a node are in order of monotonic increases memory addresses
1110 static void __init
adjust_zone_range_for_zone_movable(int nid
,
1111 unsigned long zone_type
,
1112 unsigned long node_end_pfn
,
1113 unsigned long *zone_start_pfn
,
1114 unsigned long *zone_end_pfn
)
1116 /* Only adjust if ZONE_MOVABLE is on this node */
1117 if (zone_movable_pfn
[nid
]) {
1118 /* Size ZONE_MOVABLE */
1119 if (zone_type
== ZONE_MOVABLE
) {
1120 *zone_start_pfn
= zone_movable_pfn
[nid
];
1121 *zone_end_pfn
= min(node_end_pfn
,
1122 arch_zone_highest_possible_pfn
[movable_zone
]);
1124 /* Adjust for ZONE_MOVABLE starting within this range */
1125 } else if (!mirrored_kernelcore
&&
1126 *zone_start_pfn
< zone_movable_pfn
[nid
] &&
1127 *zone_end_pfn
> zone_movable_pfn
[nid
]) {
1128 *zone_end_pfn
= zone_movable_pfn
[nid
];
1130 /* Check if this whole range is within ZONE_MOVABLE */
1131 } else if (*zone_start_pfn
>= zone_movable_pfn
[nid
])
1132 *zone_start_pfn
= *zone_end_pfn
;
1137 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
1138 * then all holes in the requested range will be accounted for.
1140 unsigned long __init
__absent_pages_in_range(int nid
,
1141 unsigned long range_start_pfn
,
1142 unsigned long range_end_pfn
)
1144 unsigned long nr_absent
= range_end_pfn
- range_start_pfn
;
1145 unsigned long start_pfn
, end_pfn
;
1148 for_each_mem_pfn_range(i
, nid
, &start_pfn
, &end_pfn
, NULL
) {
1149 start_pfn
= clamp(start_pfn
, range_start_pfn
, range_end_pfn
);
1150 end_pfn
= clamp(end_pfn
, range_start_pfn
, range_end_pfn
);
1151 nr_absent
-= end_pfn
- start_pfn
;
1157 * absent_pages_in_range - Return number of page frames in holes within a range
1158 * @start_pfn: The start PFN to start searching for holes
1159 * @end_pfn: The end PFN to stop searching for holes
1161 * Return: the number of pages frames in memory holes within a range.
1163 unsigned long __init
absent_pages_in_range(unsigned long start_pfn
,
1164 unsigned long end_pfn
)
1166 return __absent_pages_in_range(MAX_NUMNODES
, start_pfn
, end_pfn
);
1169 /* Return the number of page frames in holes in a zone on a node */
1170 static unsigned long __init
zone_absent_pages_in_node(int nid
,
1171 unsigned long zone_type
,
1172 unsigned long zone_start_pfn
,
1173 unsigned long zone_end_pfn
)
1175 unsigned long nr_absent
;
1177 /* zone is empty, we don't have any absent pages */
1178 if (zone_start_pfn
== zone_end_pfn
)
1181 nr_absent
= __absent_pages_in_range(nid
, zone_start_pfn
, zone_end_pfn
);
1184 * ZONE_MOVABLE handling.
1185 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
1188 if (mirrored_kernelcore
&& zone_movable_pfn
[nid
]) {
1189 unsigned long start_pfn
, end_pfn
;
1190 struct memblock_region
*r
;
1192 for_each_mem_region(r
) {
1193 start_pfn
= clamp(memblock_region_memory_base_pfn(r
),
1194 zone_start_pfn
, zone_end_pfn
);
1195 end_pfn
= clamp(memblock_region_memory_end_pfn(r
),
1196 zone_start_pfn
, zone_end_pfn
);
1198 if (zone_type
== ZONE_MOVABLE
&&
1199 memblock_is_mirror(r
))
1200 nr_absent
+= end_pfn
- start_pfn
;
1202 if (zone_type
== ZONE_NORMAL
&&
1203 !memblock_is_mirror(r
))
1204 nr_absent
+= end_pfn
- start_pfn
;
1212 * Return the number of pages a zone spans in a node, including holes
1213 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
1215 static unsigned long __init
zone_spanned_pages_in_node(int nid
,
1216 unsigned long zone_type
,
1217 unsigned long node_start_pfn
,
1218 unsigned long node_end_pfn
,
1219 unsigned long *zone_start_pfn
,
1220 unsigned long *zone_end_pfn
)
1222 unsigned long zone_low
= arch_zone_lowest_possible_pfn
[zone_type
];
1223 unsigned long zone_high
= arch_zone_highest_possible_pfn
[zone_type
];
1225 /* Get the start and end of the zone */
1226 *zone_start_pfn
= clamp(node_start_pfn
, zone_low
, zone_high
);
1227 *zone_end_pfn
= clamp(node_end_pfn
, zone_low
, zone_high
);
1228 adjust_zone_range_for_zone_movable(nid
, zone_type
, node_end_pfn
,
1229 zone_start_pfn
, zone_end_pfn
);
1231 /* Check that this node has pages within the zone's required range */
1232 if (*zone_end_pfn
< node_start_pfn
|| *zone_start_pfn
> node_end_pfn
)
1235 /* Move the zone boundaries inside the node if necessary */
1236 *zone_end_pfn
= min(*zone_end_pfn
, node_end_pfn
);
1237 *zone_start_pfn
= max(*zone_start_pfn
, node_start_pfn
);
1239 /* Return the spanned pages */
1240 return *zone_end_pfn
- *zone_start_pfn
;
1243 static void __init
reset_memoryless_node_totalpages(struct pglist_data
*pgdat
)
1247 for (z
= pgdat
->node_zones
; z
< pgdat
->node_zones
+ MAX_NR_ZONES
; z
++) {
1248 z
->zone_start_pfn
= 0;
1249 z
->spanned_pages
= 0;
1250 z
->present_pages
= 0;
1251 #if defined(CONFIG_MEMORY_HOTPLUG)
1252 z
->present_early_pages
= 0;
1256 pgdat
->node_spanned_pages
= 0;
1257 pgdat
->node_present_pages
= 0;
1258 pr_debug("On node %d totalpages: 0\n", pgdat
->node_id
);
1261 static void __init
calculate_node_totalpages(struct pglist_data
*pgdat
,
1262 unsigned long node_start_pfn
,
1263 unsigned long node_end_pfn
)
1265 unsigned long realtotalpages
= 0, totalpages
= 0;
1268 for (i
= 0; i
< MAX_NR_ZONES
; i
++) {
1269 struct zone
*zone
= pgdat
->node_zones
+ i
;
1270 unsigned long zone_start_pfn
, zone_end_pfn
;
1271 unsigned long spanned
, absent
;
1272 unsigned long real_size
;
1274 spanned
= zone_spanned_pages_in_node(pgdat
->node_id
, i
,
1279 absent
= zone_absent_pages_in_node(pgdat
->node_id
, i
,
1283 real_size
= spanned
- absent
;
1286 zone
->zone_start_pfn
= zone_start_pfn
;
1288 zone
->zone_start_pfn
= 0;
1289 zone
->spanned_pages
= spanned
;
1290 zone
->present_pages
= real_size
;
1291 #if defined(CONFIG_MEMORY_HOTPLUG)
1292 zone
->present_early_pages
= real_size
;
1295 totalpages
+= spanned
;
1296 realtotalpages
+= real_size
;
1299 pgdat
->node_spanned_pages
= totalpages
;
1300 pgdat
->node_present_pages
= realtotalpages
;
1301 pr_debug("On node %d totalpages: %lu\n", pgdat
->node_id
, realtotalpages
);
1304 static unsigned long __init
calc_memmap_size(unsigned long spanned_pages
,
1305 unsigned long present_pages
)
1307 unsigned long pages
= spanned_pages
;
1310 * Provide a more accurate estimation if there are holes within
1311 * the zone and SPARSEMEM is in use. If there are holes within the
1312 * zone, each populated memory region may cost us one or two extra
1313 * memmap pages due to alignment because memmap pages for each
1314 * populated regions may not be naturally aligned on page boundary.
1315 * So the (present_pages >> 4) heuristic is a tradeoff for that.
1317 if (spanned_pages
> present_pages
+ (present_pages
>> 4) &&
1318 IS_ENABLED(CONFIG_SPARSEMEM
))
1319 pages
= present_pages
;
1321 return PAGE_ALIGN(pages
* sizeof(struct page
)) >> PAGE_SHIFT
;
1324 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1325 static void pgdat_init_split_queue(struct pglist_data
*pgdat
)
1327 struct deferred_split
*ds_queue
= &pgdat
->deferred_split_queue
;
1329 spin_lock_init(&ds_queue
->split_queue_lock
);
1330 INIT_LIST_HEAD(&ds_queue
->split_queue
);
1331 ds_queue
->split_queue_len
= 0;
1334 static void pgdat_init_split_queue(struct pglist_data
*pgdat
) {}
1337 #ifdef CONFIG_COMPACTION
1338 static void pgdat_init_kcompactd(struct pglist_data
*pgdat
)
1340 init_waitqueue_head(&pgdat
->kcompactd_wait
);
1343 static void pgdat_init_kcompactd(struct pglist_data
*pgdat
) {}
1346 static void __meminit
pgdat_init_internals(struct pglist_data
*pgdat
)
1350 pgdat_resize_init(pgdat
);
1351 pgdat_kswapd_lock_init(pgdat
);
1353 pgdat_init_split_queue(pgdat
);
1354 pgdat_init_kcompactd(pgdat
);
1356 init_waitqueue_head(&pgdat
->kswapd_wait
);
1357 init_waitqueue_head(&pgdat
->pfmemalloc_wait
);
1359 for (i
= 0; i
< NR_VMSCAN_THROTTLE
; i
++)
1360 init_waitqueue_head(&pgdat
->reclaim_wait
[i
]);
1362 pgdat_page_ext_init(pgdat
);
1363 lruvec_init(&pgdat
->__lruvec
);
1366 static void __meminit
zone_init_internals(struct zone
*zone
, enum zone_type idx
, int nid
,
1367 unsigned long remaining_pages
)
1369 atomic_long_set(&zone
->managed_pages
, remaining_pages
);
1370 zone_set_nid(zone
, nid
);
1371 zone
->name
= zone_names
[idx
];
1372 zone
->zone_pgdat
= NODE_DATA(nid
);
1373 spin_lock_init(&zone
->lock
);
1374 zone_seqlock_init(zone
);
1375 zone_pcp_init(zone
);
1378 static void __meminit
zone_init_free_lists(struct zone
*zone
)
1380 unsigned int order
, t
;
1381 for_each_migratetype_order(order
, t
) {
1382 INIT_LIST_HEAD(&zone
->free_area
[order
].free_list
[t
]);
1383 zone
->free_area
[order
].nr_free
= 0;
1386 #ifdef CONFIG_UNACCEPTED_MEMORY
1387 INIT_LIST_HEAD(&zone
->unaccepted_pages
);
1391 void __meminit
init_currently_empty_zone(struct zone
*zone
,
1392 unsigned long zone_start_pfn
,
1395 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
1396 int zone_idx
= zone_idx(zone
) + 1;
1398 if (zone_idx
> pgdat
->nr_zones
)
1399 pgdat
->nr_zones
= zone_idx
;
1401 zone
->zone_start_pfn
= zone_start_pfn
;
1403 mminit_dprintk(MMINIT_TRACE
, "memmap_init",
1404 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
1406 (unsigned long)zone_idx(zone
),
1407 zone_start_pfn
, (zone_start_pfn
+ size
));
1409 zone_init_free_lists(zone
);
1410 zone
->initialized
= 1;
1413 #ifndef CONFIG_SPARSEMEM
1415 * Calculate the size of the zone->blockflags rounded to an unsigned long
1416 * Start by making sure zonesize is a multiple of pageblock_order by rounding
1417 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
1418 * round what is now in bits to nearest long in bits, then return it in
1421 static unsigned long __init
usemap_size(unsigned long zone_start_pfn
, unsigned long zonesize
)
1423 unsigned long usemapsize
;
1425 zonesize
+= zone_start_pfn
& (pageblock_nr_pages
-1);
1426 usemapsize
= roundup(zonesize
, pageblock_nr_pages
);
1427 usemapsize
= usemapsize
>> pageblock_order
;
1428 usemapsize
*= NR_PAGEBLOCK_BITS
;
1429 usemapsize
= roundup(usemapsize
, BITS_PER_LONG
);
1431 return usemapsize
/ BITS_PER_BYTE
;
1434 static void __ref
setup_usemap(struct zone
*zone
)
1436 unsigned long usemapsize
= usemap_size(zone
->zone_start_pfn
,
1437 zone
->spanned_pages
);
1438 zone
->pageblock_flags
= NULL
;
1440 zone
->pageblock_flags
=
1441 memblock_alloc_node(usemapsize
, SMP_CACHE_BYTES
,
1443 if (!zone
->pageblock_flags
)
1444 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
1445 usemapsize
, zone
->name
, zone_to_nid(zone
));
1449 static inline void setup_usemap(struct zone
*zone
) {}
1450 #endif /* CONFIG_SPARSEMEM */
1452 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
1454 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
1455 void __init
set_pageblock_order(void)
1457 unsigned int order
= MAX_ORDER
;
1459 /* Check that pageblock_nr_pages has not already been setup */
1460 if (pageblock_order
)
1463 /* Don't let pageblocks exceed the maximum allocation granularity. */
1464 if (HPAGE_SHIFT
> PAGE_SHIFT
&& HUGETLB_PAGE_ORDER
< order
)
1465 order
= HUGETLB_PAGE_ORDER
;
1468 * Assume the largest contiguous order of interest is a huge page.
1469 * This value may be variable depending on boot parameters on IA64 and
1472 pageblock_order
= order
;
1474 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1477 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
1478 * is unused as pageblock_order is set at compile-time. See
1479 * include/linux/pageblock-flags.h for the values of pageblock_order based on
1482 void __init
set_pageblock_order(void)
1486 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1489 * Set up the zone data structures
1490 * - init pgdat internals
1491 * - init all zones belonging to this node
1493 * NOTE: this function is only called during memory hotplug
1495 #ifdef CONFIG_MEMORY_HOTPLUG
1496 void __ref
free_area_init_core_hotplug(struct pglist_data
*pgdat
)
1498 int nid
= pgdat
->node_id
;
1502 pgdat_init_internals(pgdat
);
1504 if (pgdat
->per_cpu_nodestats
== &boot_nodestats
)
1505 pgdat
->per_cpu_nodestats
= alloc_percpu(struct per_cpu_nodestat
);
1508 * Reset the nr_zones, order and highest_zoneidx before reuse.
1509 * Note that kswapd will init kswapd_highest_zoneidx properly
1510 * when it starts in the near future.
1512 pgdat
->nr_zones
= 0;
1513 pgdat
->kswapd_order
= 0;
1514 pgdat
->kswapd_highest_zoneidx
= 0;
1515 pgdat
->node_start_pfn
= 0;
1516 pgdat
->node_present_pages
= 0;
1518 for_each_online_cpu(cpu
) {
1519 struct per_cpu_nodestat
*p
;
1521 p
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
);
1522 memset(p
, 0, sizeof(*p
));
1526 * When memory is hot-added, all the memory is in offline state. So
1527 * clear all zones' present_pages and managed_pages because they will
1528 * be updated in online_pages() and offline_pages().
1530 for (z
= 0; z
< MAX_NR_ZONES
; z
++) {
1531 struct zone
*zone
= pgdat
->node_zones
+ z
;
1533 zone
->present_pages
= 0;
1534 zone_init_internals(zone
, z
, nid
, 0);
1540 * Set up the zone data structures:
1541 * - mark all pages reserved
1542 * - mark all memory queues empty
1543 * - clear the memory bitmaps
1545 * NOTE: pgdat should get zeroed by caller.
1546 * NOTE: this function is only called during early init.
1548 static void __init
free_area_init_core(struct pglist_data
*pgdat
)
1551 int nid
= pgdat
->node_id
;
1553 pgdat_init_internals(pgdat
);
1554 pgdat
->per_cpu_nodestats
= &boot_nodestats
;
1556 for (j
= 0; j
< MAX_NR_ZONES
; j
++) {
1557 struct zone
*zone
= pgdat
->node_zones
+ j
;
1558 unsigned long size
, freesize
, memmap_pages
;
1560 size
= zone
->spanned_pages
;
1561 freesize
= zone
->present_pages
;
1564 * Adjust freesize so that it accounts for how much memory
1565 * is used by this zone for memmap. This affects the watermark
1566 * and per-cpu initialisations
1568 memmap_pages
= calc_memmap_size(size
, freesize
);
1569 if (!is_highmem_idx(j
)) {
1570 if (freesize
>= memmap_pages
) {
1571 freesize
-= memmap_pages
;
1573 pr_debug(" %s zone: %lu pages used for memmap\n",
1574 zone_names
[j
], memmap_pages
);
1576 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n",
1577 zone_names
[j
], memmap_pages
, freesize
);
1580 /* Account for reserved pages */
1581 if (j
== 0 && freesize
> dma_reserve
) {
1582 freesize
-= dma_reserve
;
1583 pr_debug(" %s zone: %lu pages reserved\n", zone_names
[0], dma_reserve
);
1586 if (!is_highmem_idx(j
))
1587 nr_kernel_pages
+= freesize
;
1588 /* Charge for highmem memmap if there are enough kernel pages */
1589 else if (nr_kernel_pages
> memmap_pages
* 2)
1590 nr_kernel_pages
-= memmap_pages
;
1591 nr_all_pages
+= freesize
;
1594 * Set an approximate value for lowmem here, it will be adjusted
1595 * when the bootmem allocator frees pages into the buddy system.
1596 * And all highmem pages will be managed by the buddy system.
1598 zone_init_internals(zone
, j
, nid
, freesize
);
1604 init_currently_empty_zone(zone
, zone
->zone_start_pfn
, size
);
1608 void __init
*memmap_alloc(phys_addr_t size
, phys_addr_t align
,
1609 phys_addr_t min_addr
, int nid
, bool exact_nid
)
1614 ptr
= memblock_alloc_exact_nid_raw(size
, align
, min_addr
,
1615 MEMBLOCK_ALLOC_ACCESSIBLE
,
1618 ptr
= memblock_alloc_try_nid_raw(size
, align
, min_addr
,
1619 MEMBLOCK_ALLOC_ACCESSIBLE
,
1622 if (ptr
&& size
> 0)
1623 page_init_poison(ptr
, size
);
1628 #ifdef CONFIG_FLATMEM
1629 static void __init
alloc_node_mem_map(struct pglist_data
*pgdat
)
1631 unsigned long __maybe_unused start
= 0;
1632 unsigned long __maybe_unused offset
= 0;
1634 /* Skip empty nodes */
1635 if (!pgdat
->node_spanned_pages
)
1638 start
= pgdat
->node_start_pfn
& ~(MAX_ORDER_NR_PAGES
- 1);
1639 offset
= pgdat
->node_start_pfn
- start
;
1640 /* ia64 gets its own node_mem_map, before this, without bootmem */
1641 if (!pgdat
->node_mem_map
) {
1642 unsigned long size
, end
;
1646 * The zone's endpoints aren't required to be MAX_ORDER
1647 * aligned but the node_mem_map endpoints must be in order
1648 * for the buddy allocator to function correctly.
1650 end
= pgdat_end_pfn(pgdat
);
1651 end
= ALIGN(end
, MAX_ORDER_NR_PAGES
);
1652 size
= (end
- start
) * sizeof(struct page
);
1653 map
= memmap_alloc(size
, SMP_CACHE_BYTES
, MEMBLOCK_LOW_LIMIT
,
1654 pgdat
->node_id
, false);
1656 panic("Failed to allocate %ld bytes for node %d memory map\n",
1657 size
, pgdat
->node_id
);
1658 pgdat
->node_mem_map
= map
+ offset
;
1660 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
1661 __func__
, pgdat
->node_id
, (unsigned long)pgdat
,
1662 (unsigned long)pgdat
->node_mem_map
);
1665 * With no DISCONTIG, the global mem_map is just set as node 0's
1667 if (pgdat
== NODE_DATA(0)) {
1668 mem_map
= NODE_DATA(0)->node_mem_map
;
1669 if (page_to_pfn(mem_map
) != pgdat
->node_start_pfn
)
1675 static inline void alloc_node_mem_map(struct pglist_data
*pgdat
) { }
1676 #endif /* CONFIG_FLATMEM */
1679 * get_pfn_range_for_nid - Return the start and end page frames for a node
1680 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
1681 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
1682 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
1684 * It returns the start and end page frame of a node based on information
1685 * provided by memblock_set_node(). If called for a node
1686 * with no available memory, the start and end PFNs will be 0.
1688 void __init
get_pfn_range_for_nid(unsigned int nid
,
1689 unsigned long *start_pfn
, unsigned long *end_pfn
)
1691 unsigned long this_start_pfn
, this_end_pfn
;
1697 for_each_mem_pfn_range(i
, nid
, &this_start_pfn
, &this_end_pfn
, NULL
) {
1698 *start_pfn
= min(*start_pfn
, this_start_pfn
);
1699 *end_pfn
= max(*end_pfn
, this_end_pfn
);
1702 if (*start_pfn
== -1UL)
1706 static void __init
free_area_init_node(int nid
)
1708 pg_data_t
*pgdat
= NODE_DATA(nid
);
1709 unsigned long start_pfn
= 0;
1710 unsigned long end_pfn
= 0;
1712 /* pg_data_t should be reset to zero when it's allocated */
1713 WARN_ON(pgdat
->nr_zones
|| pgdat
->kswapd_highest_zoneidx
);
1715 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
1717 pgdat
->node_id
= nid
;
1718 pgdat
->node_start_pfn
= start_pfn
;
1719 pgdat
->per_cpu_nodestats
= NULL
;
1721 if (start_pfn
!= end_pfn
) {
1722 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid
,
1723 (u64
)start_pfn
<< PAGE_SHIFT
,
1724 end_pfn
? ((u64
)end_pfn
<< PAGE_SHIFT
) - 1 : 0);
1726 calculate_node_totalpages(pgdat
, start_pfn
, end_pfn
);
1728 pr_info("Initmem setup node %d as memoryless\n", nid
);
1730 reset_memoryless_node_totalpages(pgdat
);
1733 alloc_node_mem_map(pgdat
);
1734 pgdat_set_deferred_range(pgdat
);
1736 free_area_init_core(pgdat
);
1737 lru_gen_init_pgdat(pgdat
);
1740 /* Any regular or high memory on that node ? */
1741 static void __init
check_for_memory(pg_data_t
*pgdat
)
1743 enum zone_type zone_type
;
1745 for (zone_type
= 0; zone_type
<= ZONE_MOVABLE
- 1; zone_type
++) {
1746 struct zone
*zone
= &pgdat
->node_zones
[zone_type
];
1747 if (populated_zone(zone
)) {
1748 if (IS_ENABLED(CONFIG_HIGHMEM
))
1749 node_set_state(pgdat
->node_id
, N_HIGH_MEMORY
);
1750 if (zone_type
<= ZONE_NORMAL
)
1751 node_set_state(pgdat
->node_id
, N_NORMAL_MEMORY
);
1757 #if MAX_NUMNODES > 1
1759 * Figure out the number of possible node ids.
1761 void __init
setup_nr_node_ids(void)
1763 unsigned int highest
;
1765 highest
= find_last_bit(node_possible_map
.bits
, MAX_NUMNODES
);
1766 nr_node_ids
= highest
+ 1;
1771 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
1772 * such cases we allow max_zone_pfn sorted in the descending order
1774 static bool arch_has_descending_max_zone_pfns(void)
1776 return IS_ENABLED(CONFIG_ARC
) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40
);
1780 * free_area_init - Initialise all pg_data_t and zone data
1781 * @max_zone_pfn: an array of max PFNs for each zone
1783 * This will call free_area_init_node() for each active node in the system.
1784 * Using the page ranges provided by memblock_set_node(), the size of each
1785 * zone in each node and their holes is calculated. If the maximum PFN
1786 * between two adjacent zones match, it is assumed that the zone is empty.
1787 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
1788 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
1789 * starts where the previous one ended. For example, ZONE_DMA32 starts
1790 * at arch_max_dma_pfn.
1792 void __init
free_area_init(unsigned long *max_zone_pfn
)
1794 unsigned long start_pfn
, end_pfn
;
1798 /* Record where the zone boundaries are */
1799 memset(arch_zone_lowest_possible_pfn
, 0,
1800 sizeof(arch_zone_lowest_possible_pfn
));
1801 memset(arch_zone_highest_possible_pfn
, 0,
1802 sizeof(arch_zone_highest_possible_pfn
));
1804 start_pfn
= PHYS_PFN(memblock_start_of_DRAM());
1805 descending
= arch_has_descending_max_zone_pfns();
1807 for (i
= 0; i
< MAX_NR_ZONES
; i
++) {
1809 zone
= MAX_NR_ZONES
- i
- 1;
1813 if (zone
== ZONE_MOVABLE
)
1816 end_pfn
= max(max_zone_pfn
[zone
], start_pfn
);
1817 arch_zone_lowest_possible_pfn
[zone
] = start_pfn
;
1818 arch_zone_highest_possible_pfn
[zone
] = end_pfn
;
1820 start_pfn
= end_pfn
;
1823 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
1824 memset(zone_movable_pfn
, 0, sizeof(zone_movable_pfn
));
1825 find_zone_movable_pfns_for_nodes();
1827 /* Print out the zone ranges */
1828 pr_info("Zone ranges:\n");
1829 for (i
= 0; i
< MAX_NR_ZONES
; i
++) {
1830 if (i
== ZONE_MOVABLE
)
1832 pr_info(" %-8s ", zone_names
[i
]);
1833 if (arch_zone_lowest_possible_pfn
[i
] ==
1834 arch_zone_highest_possible_pfn
[i
])
1837 pr_cont("[mem %#018Lx-%#018Lx]\n",
1838 (u64
)arch_zone_lowest_possible_pfn
[i
]
1840 ((u64
)arch_zone_highest_possible_pfn
[i
]
1841 << PAGE_SHIFT
) - 1);
1844 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
1845 pr_info("Movable zone start for each node\n");
1846 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
1847 if (zone_movable_pfn
[i
])
1848 pr_info(" Node %d: %#018Lx\n", i
,
1849 (u64
)zone_movable_pfn
[i
] << PAGE_SHIFT
);
1853 * Print out the early node map, and initialize the
1854 * subsection-map relative to active online memory ranges to
1855 * enable future "sub-section" extensions of the memory map.
1857 pr_info("Early memory node ranges\n");
1858 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start_pfn
, &end_pfn
, &nid
) {
1859 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid
,
1860 (u64
)start_pfn
<< PAGE_SHIFT
,
1861 ((u64
)end_pfn
<< PAGE_SHIFT
) - 1);
1862 subsection_map_init(start_pfn
, end_pfn
- start_pfn
);
1865 /* Initialise every node */
1866 mminit_verify_pageflags_layout();
1867 setup_nr_node_ids();
1868 set_pageblock_order();
1870 for_each_node(nid
) {
1873 if (!node_online(nid
)) {
1874 pr_info("Initializing node %d as memoryless\n", nid
);
1876 /* Allocator not initialized yet */
1877 pgdat
= arch_alloc_nodedata(nid
);
1879 panic("Cannot allocate %zuB for node %d.\n",
1880 sizeof(*pgdat
), nid
);
1881 arch_refresh_nodedata(nid
, pgdat
);
1882 free_area_init_node(nid
);
1885 * We do not want to confuse userspace by sysfs
1886 * files/directories for node without any memory
1887 * attached to it, so this node is not marked as
1888 * N_MEMORY and not marked online so that no sysfs
1889 * hierarchy will be created via register_one_node for
1890 * it. The pgdat will get fully initialized by
1891 * hotadd_init_pgdat() when memory is hotplugged into
1897 pgdat
= NODE_DATA(nid
);
1898 free_area_init_node(nid
);
1900 /* Any memory on that node */
1901 if (pgdat
->node_present_pages
)
1902 node_set_state(nid
, N_MEMORY
);
1903 check_for_memory(pgdat
);
1908 /* disable hash distribution for systems with a single node */
1913 * node_map_pfn_alignment - determine the maximum internode alignment
1915 * This function should be called after node map is populated and sorted.
1916 * It calculates the maximum power of two alignment which can distinguish
1919 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
1920 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
1921 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
1922 * shifted, 1GiB is enough and this function will indicate so.
1924 * This is used to test whether pfn -> nid mapping of the chosen memory
1925 * model has fine enough granularity to avoid incorrect mapping for the
1926 * populated node map.
1928 * Return: the determined alignment in pfn's. 0 if there is no alignment
1929 * requirement (single node).
1931 unsigned long __init
node_map_pfn_alignment(void)
1933 unsigned long accl_mask
= 0, last_end
= 0;
1934 unsigned long start
, end
, mask
;
1935 int last_nid
= NUMA_NO_NODE
;
1938 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start
, &end
, &nid
) {
1939 if (!start
|| last_nid
< 0 || last_nid
== nid
) {
1946 * Start with a mask granular enough to pin-point to the
1947 * start pfn and tick off bits one-by-one until it becomes
1948 * too coarse to separate the current node from the last.
1950 mask
= ~((1 << __ffs(start
)) - 1);
1951 while (mask
&& last_end
<= (start
& (mask
<< 1)))
1954 /* accumulate all internode masks */
1958 /* convert mask to number of pages */
1959 return ~accl_mask
+ 1;
1962 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1963 static void __init
deferred_free_range(unsigned long pfn
,
1964 unsigned long nr_pages
)
1972 page
= pfn_to_page(pfn
);
1974 /* Free a large naturally-aligned chunk if possible */
1975 if (nr_pages
== MAX_ORDER_NR_PAGES
&& IS_MAX_ORDER_ALIGNED(pfn
)) {
1976 for (i
= 0; i
< nr_pages
; i
+= pageblock_nr_pages
)
1977 set_pageblock_migratetype(page
+ i
, MIGRATE_MOVABLE
);
1978 __free_pages_core(page
, MAX_ORDER
);
1982 /* Accept chunks smaller than MAX_ORDER upfront */
1983 accept_memory(PFN_PHYS(pfn
), PFN_PHYS(pfn
+ nr_pages
));
1985 for (i
= 0; i
< nr_pages
; i
++, page
++, pfn
++) {
1986 if (pageblock_aligned(pfn
))
1987 set_pageblock_migratetype(page
, MIGRATE_MOVABLE
);
1988 __free_pages_core(page
, 0);
1992 /* Completion tracking for deferred_init_memmap() threads */
1993 static atomic_t pgdat_init_n_undone __initdata
;
1994 static __initdata
DECLARE_COMPLETION(pgdat_init_all_done_comp
);
1996 static inline void __init
pgdat_init_report_one_done(void)
1998 if (atomic_dec_and_test(&pgdat_init_n_undone
))
1999 complete(&pgdat_init_all_done_comp
);
2003 * Returns true if page needs to be initialized or freed to buddy allocator.
2005 * We check if a current MAX_ORDER block is valid by only checking the validity
2008 static inline bool __init
deferred_pfn_valid(unsigned long pfn
)
2010 if (IS_MAX_ORDER_ALIGNED(pfn
) && !pfn_valid(pfn
))
2016 * Free pages to buddy allocator. Try to free aligned pages in
2017 * MAX_ORDER_NR_PAGES sizes.
2019 static void __init
deferred_free_pages(unsigned long pfn
,
2020 unsigned long end_pfn
)
2022 unsigned long nr_free
= 0;
2024 for (; pfn
< end_pfn
; pfn
++) {
2025 if (!deferred_pfn_valid(pfn
)) {
2026 deferred_free_range(pfn
- nr_free
, nr_free
);
2028 } else if (IS_MAX_ORDER_ALIGNED(pfn
)) {
2029 deferred_free_range(pfn
- nr_free
, nr_free
);
2035 /* Free the last block of pages to allocator */
2036 deferred_free_range(pfn
- nr_free
, nr_free
);
2040 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
2041 * by performing it only once every MAX_ORDER_NR_PAGES.
2042 * Return number of pages initialized.
2044 static unsigned long __init
deferred_init_pages(struct zone
*zone
,
2046 unsigned long end_pfn
)
2048 int nid
= zone_to_nid(zone
);
2049 unsigned long nr_pages
= 0;
2050 int zid
= zone_idx(zone
);
2051 struct page
*page
= NULL
;
2053 for (; pfn
< end_pfn
; pfn
++) {
2054 if (!deferred_pfn_valid(pfn
)) {
2057 } else if (!page
|| IS_MAX_ORDER_ALIGNED(pfn
)) {
2058 page
= pfn_to_page(pfn
);
2062 __init_single_page(page
, pfn
, zid
, nid
);
2069 * This function is meant to pre-load the iterator for the zone init.
2070 * Specifically it walks through the ranges until we are caught up to the
2071 * first_init_pfn value and exits there. If we never encounter the value we
2072 * return false indicating there are no valid ranges left.
2075 deferred_init_mem_pfn_range_in_zone(u64
*i
, struct zone
*zone
,
2076 unsigned long *spfn
, unsigned long *epfn
,
2077 unsigned long first_init_pfn
)
2082 * Start out by walking through the ranges in this zone that have
2083 * already been initialized. We don't need to do anything with them
2084 * so we just need to flush them out of the system.
2086 for_each_free_mem_pfn_range_in_zone(j
, zone
, spfn
, epfn
) {
2087 if (*epfn
<= first_init_pfn
)
2089 if (*spfn
< first_init_pfn
)
2090 *spfn
= first_init_pfn
;
2099 * Initialize and free pages. We do it in two loops: first we initialize
2100 * struct page, then free to buddy allocator, because while we are
2101 * freeing pages we can access pages that are ahead (computing buddy
2102 * page in __free_one_page()).
2104 * In order to try and keep some memory in the cache we have the loop
2105 * broken along max page order boundaries. This way we will not cause
2106 * any issues with the buddy page computation.
2108 static unsigned long __init
2109 deferred_init_maxorder(u64
*i
, struct zone
*zone
, unsigned long *start_pfn
,
2110 unsigned long *end_pfn
)
2112 unsigned long mo_pfn
= ALIGN(*start_pfn
+ 1, MAX_ORDER_NR_PAGES
);
2113 unsigned long spfn
= *start_pfn
, epfn
= *end_pfn
;
2114 unsigned long nr_pages
= 0;
2117 /* First we loop through and initialize the page values */
2118 for_each_free_mem_pfn_range_in_zone_from(j
, zone
, start_pfn
, end_pfn
) {
2121 if (mo_pfn
<= *start_pfn
)
2124 t
= min(mo_pfn
, *end_pfn
);
2125 nr_pages
+= deferred_init_pages(zone
, *start_pfn
, t
);
2127 if (mo_pfn
< *end_pfn
) {
2128 *start_pfn
= mo_pfn
;
2133 /* Reset values and now loop through freeing pages as needed */
2136 for_each_free_mem_pfn_range_in_zone_from(j
, zone
, &spfn
, &epfn
) {
2142 t
= min(mo_pfn
, epfn
);
2143 deferred_free_pages(spfn
, t
);
2153 deferred_init_memmap_chunk(unsigned long start_pfn
, unsigned long end_pfn
,
2156 unsigned long spfn
, epfn
;
2157 struct zone
*zone
= arg
;
2160 deferred_init_mem_pfn_range_in_zone(&i
, zone
, &spfn
, &epfn
, start_pfn
);
2163 * Initialize and free pages in MAX_ORDER sized increments so that we
2164 * can avoid introducing any issues with the buddy allocator.
2166 while (spfn
< end_pfn
) {
2167 deferred_init_maxorder(&i
, zone
, &spfn
, &epfn
);
2172 /* An arch may override for more concurrency. */
2174 deferred_page_init_max_threads(const struct cpumask
*node_cpumask
)
2179 /* Initialise remaining memory on a node */
2180 static int __init
deferred_init_memmap(void *data
)
2182 pg_data_t
*pgdat
= data
;
2183 const struct cpumask
*cpumask
= cpumask_of_node(pgdat
->node_id
);
2184 unsigned long spfn
= 0, epfn
= 0;
2185 unsigned long first_init_pfn
, flags
;
2186 unsigned long start
= jiffies
;
2188 int zid
, max_threads
;
2191 /* Bind memory initialisation thread to a local node if possible */
2192 if (!cpumask_empty(cpumask
))
2193 set_cpus_allowed_ptr(current
, cpumask
);
2195 pgdat_resize_lock(pgdat
, &flags
);
2196 first_init_pfn
= pgdat
->first_deferred_pfn
;
2197 if (first_init_pfn
== ULONG_MAX
) {
2198 pgdat_resize_unlock(pgdat
, &flags
);
2199 pgdat_init_report_one_done();
2203 /* Sanity check boundaries */
2204 BUG_ON(pgdat
->first_deferred_pfn
< pgdat
->node_start_pfn
);
2205 BUG_ON(pgdat
->first_deferred_pfn
> pgdat_end_pfn(pgdat
));
2206 pgdat
->first_deferred_pfn
= ULONG_MAX
;
2209 * Once we unlock here, the zone cannot be grown anymore, thus if an
2210 * interrupt thread must allocate this early in boot, zone must be
2211 * pre-grown prior to start of deferred page initialization.
2213 pgdat_resize_unlock(pgdat
, &flags
);
2215 /* Only the highest zone is deferred so find it */
2216 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
2217 zone
= pgdat
->node_zones
+ zid
;
2218 if (first_init_pfn
< zone_end_pfn(zone
))
2222 /* If the zone is empty somebody else may have cleared out the zone */
2223 if (!deferred_init_mem_pfn_range_in_zone(&i
, zone
, &spfn
, &epfn
,
2227 max_threads
= deferred_page_init_max_threads(cpumask
);
2229 while (spfn
< epfn
) {
2230 unsigned long epfn_align
= ALIGN(epfn
, PAGES_PER_SECTION
);
2231 struct padata_mt_job job
= {
2232 .thread_fn
= deferred_init_memmap_chunk
,
2235 .size
= epfn_align
- spfn
,
2236 .align
= PAGES_PER_SECTION
,
2237 .min_chunk
= PAGES_PER_SECTION
,
2238 .max_threads
= max_threads
,
2241 padata_do_multithreaded(&job
);
2242 deferred_init_mem_pfn_range_in_zone(&i
, zone
, &spfn
, &epfn
,
2246 /* Sanity check that the next zone really is unpopulated */
2247 WARN_ON(++zid
< MAX_NR_ZONES
&& populated_zone(++zone
));
2249 pr_info("node %d deferred pages initialised in %ums\n",
2250 pgdat
->node_id
, jiffies_to_msecs(jiffies
- start
));
2252 pgdat_init_report_one_done();
2257 * If this zone has deferred pages, try to grow it by initializing enough
2258 * deferred pages to satisfy the allocation specified by order, rounded up to
2259 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
2260 * of SECTION_SIZE bytes by initializing struct pages in increments of
2261 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2263 * Return true when zone was grown, otherwise return false. We return true even
2264 * when we grow less than requested, to let the caller decide if there are
2265 * enough pages to satisfy the allocation.
2267 * Note: We use noinline because this function is needed only during boot, and
2268 * it is called from a __ref function _deferred_grow_zone. This way we are
2269 * making sure that it is not inlined into permanent text section.
2271 bool __init
deferred_grow_zone(struct zone
*zone
, unsigned int order
)
2273 unsigned long nr_pages_needed
= ALIGN(1 << order
, PAGES_PER_SECTION
);
2274 pg_data_t
*pgdat
= zone
->zone_pgdat
;
2275 unsigned long first_deferred_pfn
= pgdat
->first_deferred_pfn
;
2276 unsigned long spfn
, epfn
, flags
;
2277 unsigned long nr_pages
= 0;
2280 /* Only the last zone may have deferred pages */
2281 if (zone_end_pfn(zone
) != pgdat_end_pfn(pgdat
))
2284 pgdat_resize_lock(pgdat
, &flags
);
2287 * If someone grew this zone while we were waiting for spinlock, return
2288 * true, as there might be enough pages already.
2290 if (first_deferred_pfn
!= pgdat
->first_deferred_pfn
) {
2291 pgdat_resize_unlock(pgdat
, &flags
);
2295 /* If the zone is empty somebody else may have cleared out the zone */
2296 if (!deferred_init_mem_pfn_range_in_zone(&i
, zone
, &spfn
, &epfn
,
2297 first_deferred_pfn
)) {
2298 pgdat
->first_deferred_pfn
= ULONG_MAX
;
2299 pgdat_resize_unlock(pgdat
, &flags
);
2300 /* Retry only once. */
2301 return first_deferred_pfn
!= ULONG_MAX
;
2305 * Initialize and free pages in MAX_ORDER sized increments so
2306 * that we can avoid introducing any issues with the buddy
2309 while (spfn
< epfn
) {
2310 /* update our first deferred PFN for this section */
2311 first_deferred_pfn
= spfn
;
2313 nr_pages
+= deferred_init_maxorder(&i
, zone
, &spfn
, &epfn
);
2314 touch_nmi_watchdog();
2316 /* We should only stop along section boundaries */
2317 if ((first_deferred_pfn
^ spfn
) < PAGES_PER_SECTION
)
2320 /* If our quota has been met we can stop here */
2321 if (nr_pages
>= nr_pages_needed
)
2325 pgdat
->first_deferred_pfn
= spfn
;
2326 pgdat_resize_unlock(pgdat
, &flags
);
2328 return nr_pages
> 0;
2331 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2334 void __init
init_cma_reserved_pageblock(struct page
*page
)
2336 unsigned i
= pageblock_nr_pages
;
2337 struct page
*p
= page
;
2340 __ClearPageReserved(p
);
2341 set_page_count(p
, 0);
2344 set_pageblock_migratetype(page
, MIGRATE_CMA
);
2345 set_page_refcounted(page
);
2346 __free_pages(page
, pageblock_order
);
2348 adjust_managed_page_count(page
, pageblock_nr_pages
);
2349 page_zone(page
)->cma_pages
+= pageblock_nr_pages
;
2353 void set_zone_contiguous(struct zone
*zone
)
2355 unsigned long block_start_pfn
= zone
->zone_start_pfn
;
2356 unsigned long block_end_pfn
;
2358 block_end_pfn
= pageblock_end_pfn(block_start_pfn
);
2359 for (; block_start_pfn
< zone_end_pfn(zone
);
2360 block_start_pfn
= block_end_pfn
,
2361 block_end_pfn
+= pageblock_nr_pages
) {
2363 block_end_pfn
= min(block_end_pfn
, zone_end_pfn(zone
));
2365 if (!__pageblock_pfn_to_page(block_start_pfn
,
2366 block_end_pfn
, zone
))
2371 /* We confirm that there is no hole */
2372 zone
->contiguous
= true;
2375 void __init
page_alloc_init_late(void)
2380 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2382 /* There will be num_node_state(N_MEMORY) threads */
2383 atomic_set(&pgdat_init_n_undone
, num_node_state(N_MEMORY
));
2384 for_each_node_state(nid
, N_MEMORY
) {
2385 kthread_run(deferred_init_memmap
, NODE_DATA(nid
), "pgdatinit%d", nid
);
2388 /* Block until all are initialised */
2389 wait_for_completion(&pgdat_init_all_done_comp
);
2392 * We initialized the rest of the deferred pages. Permanently disable
2393 * on-demand struct page initialization.
2395 static_branch_disable(&deferred_pages
);
2397 /* Reinit limits that are based on free pages after the kernel is up */
2398 files_maxfiles_init();
2403 /* Discard memblock private memory */
2406 for_each_node_state(nid
, N_MEMORY
)
2407 shuffle_free_memory(NODE_DATA(nid
));
2409 for_each_populated_zone(zone
)
2410 set_zone_contiguous(zone
);
2412 /* Initialize page ext after all struct pages are initialized. */
2413 if (deferred_struct_pages
)
2416 page_alloc_sysctl_init();
2419 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
2421 * Returns the number of pages that arch has reserved but
2422 * is not known to alloc_large_system_hash().
2424 static unsigned long __init
arch_reserved_kernel_pages(void)
2431 * Adaptive scale is meant to reduce sizes of hash tables on large memory
2432 * machines. As memory size is increased the scale is also increased but at
2433 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
2434 * quadruples the scale is increased by one, which means the size of hash table
2435 * only doubles, instead of quadrupling as well.
2436 * Because 32-bit systems cannot have large physical memory, where this scaling
2437 * makes sense, it is disabled on such platforms.
2439 #if __BITS_PER_LONG > 32
2440 #define ADAPT_SCALE_BASE (64ul << 30)
2441 #define ADAPT_SCALE_SHIFT 2
2442 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
2446 * allocate a large system hash table from bootmem
2447 * - it is assumed that the hash table must contain an exact power-of-2
2448 * quantity of entries
2449 * - limit is the number of hash buckets, not the total allocation size
2451 void *__init
alloc_large_system_hash(const char *tablename
,
2452 unsigned long bucketsize
,
2453 unsigned long numentries
,
2456 unsigned int *_hash_shift
,
2457 unsigned int *_hash_mask
,
2458 unsigned long low_limit
,
2459 unsigned long high_limit
)
2461 unsigned long long max
= high_limit
;
2462 unsigned long log2qty
, size
;
2468 /* allow the kernel cmdline to have a say */
2470 /* round applicable memory size up to nearest megabyte */
2471 numentries
= nr_kernel_pages
;
2472 numentries
-= arch_reserved_kernel_pages();
2474 /* It isn't necessary when PAGE_SIZE >= 1MB */
2475 if (PAGE_SIZE
< SZ_1M
)
2476 numentries
= round_up(numentries
, SZ_1M
/ PAGE_SIZE
);
2478 #if __BITS_PER_LONG > 32
2480 unsigned long adapt
;
2482 for (adapt
= ADAPT_SCALE_NPAGES
; adapt
< numentries
;
2483 adapt
<<= ADAPT_SCALE_SHIFT
)
2488 /* limit to 1 bucket per 2^scale bytes of low memory */
2489 if (scale
> PAGE_SHIFT
)
2490 numentries
>>= (scale
- PAGE_SHIFT
);
2492 numentries
<<= (PAGE_SHIFT
- scale
);
2494 if (unlikely((numentries
* bucketsize
) < PAGE_SIZE
))
2495 numentries
= PAGE_SIZE
/ bucketsize
;
2497 numentries
= roundup_pow_of_two(numentries
);
2499 /* limit allocation size to 1/16 total memory by default */
2501 max
= ((unsigned long long)nr_all_pages
<< PAGE_SHIFT
) >> 4;
2502 do_div(max
, bucketsize
);
2504 max
= min(max
, 0x80000000ULL
);
2506 if (numentries
< low_limit
)
2507 numentries
= low_limit
;
2508 if (numentries
> max
)
2511 log2qty
= ilog2(numentries
);
2513 gfp_flags
= (flags
& HASH_ZERO
) ? GFP_ATOMIC
| __GFP_ZERO
: GFP_ATOMIC
;
2516 size
= bucketsize
<< log2qty
;
2517 if (flags
& HASH_EARLY
) {
2518 if (flags
& HASH_ZERO
)
2519 table
= memblock_alloc(size
, SMP_CACHE_BYTES
);
2521 table
= memblock_alloc_raw(size
,
2523 } else if (get_order(size
) > MAX_ORDER
|| hashdist
) {
2524 table
= vmalloc_huge(size
, gfp_flags
);
2527 huge
= is_vm_area_hugepages(table
);
2530 * If bucketsize is not a power-of-two, we may free
2531 * some pages at the end of hash table which
2532 * alloc_pages_exact() automatically does
2534 table
= alloc_pages_exact(size
, gfp_flags
);
2535 kmemleak_alloc(table
, size
, 1, gfp_flags
);
2537 } while (!table
&& size
> PAGE_SIZE
&& --log2qty
);
2540 panic("Failed to allocate %s hash table\n", tablename
);
2542 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
2543 tablename
, 1UL << log2qty
, ilog2(size
) - PAGE_SHIFT
, size
,
2544 virt
? (huge
? "vmalloc hugepage" : "vmalloc") : "linear");
2547 *_hash_shift
= log2qty
;
2549 *_hash_mask
= (1 << log2qty
) - 1;
2555 * set_dma_reserve - set the specified number of pages reserved in the first zone
2556 * @new_dma_reserve: The number of pages to mark reserved
2558 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
2559 * In the DMA zone, a significant percentage may be consumed by kernel image
2560 * and other unfreeable allocations which can skew the watermarks badly. This
2561 * function may optionally be used to account for unfreeable pages in the
2562 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
2563 * smaller per-cpu batchsize.
2565 void __init
set_dma_reserve(unsigned long new_dma_reserve
)
2567 dma_reserve
= new_dma_reserve
;
2570 void __init
memblock_free_pages(struct page
*page
, unsigned long pfn
,
2574 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT
)) {
2575 int nid
= early_pfn_to_nid(pfn
);
2577 if (!early_page_initialised(pfn
, nid
))
2581 if (!kmsan_memblock_free_pages(page
, order
)) {
2582 /* KMSAN will take care of these pages. */
2585 __free_pages_core(page
, order
);
2588 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON
, init_on_alloc
);
2589 EXPORT_SYMBOL(init_on_alloc
);
2591 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON
, init_on_free
);
2592 EXPORT_SYMBOL(init_on_free
);
2594 static bool _init_on_alloc_enabled_early __read_mostly
2595 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON
);
2596 static int __init
early_init_on_alloc(char *buf
)
2599 return kstrtobool(buf
, &_init_on_alloc_enabled_early
);
2601 early_param("init_on_alloc", early_init_on_alloc
);
2603 static bool _init_on_free_enabled_early __read_mostly
2604 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON
);
2605 static int __init
early_init_on_free(char *buf
)
2607 return kstrtobool(buf
, &_init_on_free_enabled_early
);
2609 early_param("init_on_free", early_init_on_free
);
2611 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM
, check_pages_enabled
);
2614 * Enable static keys related to various memory debugging and hardening options.
2615 * Some override others, and depend on early params that are evaluated in the
2616 * order of appearance. So we need to first gather the full picture of what was
2617 * enabled, and then make decisions.
2619 static void __init
mem_debugging_and_hardening_init(void)
2621 bool page_poisoning_requested
= false;
2622 bool want_check_pages
= false;
2624 #ifdef CONFIG_PAGE_POISONING
2626 * Page poisoning is debug page alloc for some arches. If
2627 * either of those options are enabled, enable poisoning.
2629 if (page_poisoning_enabled() ||
2630 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC
) &&
2631 debug_pagealloc_enabled())) {
2632 static_branch_enable(&_page_poisoning_enabled
);
2633 page_poisoning_requested
= true;
2634 want_check_pages
= true;
2638 if ((_init_on_alloc_enabled_early
|| _init_on_free_enabled_early
) &&
2639 page_poisoning_requested
) {
2640 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
2641 "will take precedence over init_on_alloc and init_on_free\n");
2642 _init_on_alloc_enabled_early
= false;
2643 _init_on_free_enabled_early
= false;
2646 if (_init_on_alloc_enabled_early
) {
2647 want_check_pages
= true;
2648 static_branch_enable(&init_on_alloc
);
2650 static_branch_disable(&init_on_alloc
);
2653 if (_init_on_free_enabled_early
) {
2654 want_check_pages
= true;
2655 static_branch_enable(&init_on_free
);
2657 static_branch_disable(&init_on_free
);
2660 if (IS_ENABLED(CONFIG_KMSAN
) &&
2661 (_init_on_alloc_enabled_early
|| _init_on_free_enabled_early
))
2662 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
2664 #ifdef CONFIG_DEBUG_PAGEALLOC
2665 if (debug_pagealloc_enabled()) {
2666 want_check_pages
= true;
2667 static_branch_enable(&_debug_pagealloc_enabled
);
2669 if (debug_guardpage_minorder())
2670 static_branch_enable(&_debug_guardpage_enabled
);
2675 * Any page debugging or hardening option also enables sanity checking
2676 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's
2679 if (!IS_ENABLED(CONFIG_DEBUG_VM
) && want_check_pages
)
2680 static_branch_enable(&check_pages_enabled
);
2683 /* Report memory auto-initialization states for this boot. */
2684 static void __init
report_meminit(void)
2688 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN
))
2689 stack
= "all(pattern)";
2690 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO
))
2691 stack
= "all(zero)";
2692 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL
))
2693 stack
= "byref_all(zero)";
2694 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF
))
2695 stack
= "byref(zero)";
2696 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER
))
2697 stack
= "__user(zero)";
2701 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
2702 stack
, want_init_on_alloc(GFP_KERNEL
) ? "on" : "off",
2703 want_init_on_free() ? "on" : "off");
2704 if (want_init_on_free())
2705 pr_info("mem auto-init: clearing system memory may take some time...\n");
2708 static void __init
mem_init_print_info(void)
2710 unsigned long physpages
, codesize
, datasize
, rosize
, bss_size
;
2711 unsigned long init_code_size
, init_data_size
;
2713 physpages
= get_num_physpages();
2714 codesize
= _etext
- _stext
;
2715 datasize
= _edata
- _sdata
;
2716 rosize
= __end_rodata
- __start_rodata
;
2717 bss_size
= __bss_stop
- __bss_start
;
2718 init_data_size
= __init_end
- __init_begin
;
2719 init_code_size
= _einittext
- _sinittext
;
2722 * Detect special cases and adjust section sizes accordingly:
2723 * 1) .init.* may be embedded into .data sections
2724 * 2) .init.text.* may be out of [__init_begin, __init_end],
2725 * please refer to arch/tile/kernel/vmlinux.lds.S.
2726 * 3) .rodata.* may be embedded into .text or .data sections.
2728 #define adj_init_size(start, end, size, pos, adj) \
2730 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
2734 adj_init_size(__init_begin
, __init_end
, init_data_size
,
2735 _sinittext
, init_code_size
);
2736 adj_init_size(_stext
, _etext
, codesize
, _sinittext
, init_code_size
);
2737 adj_init_size(_sdata
, _edata
, datasize
, __init_begin
, init_data_size
);
2738 adj_init_size(_stext
, _etext
, codesize
, __start_rodata
, rosize
);
2739 adj_init_size(_sdata
, _edata
, datasize
, __start_rodata
, rosize
);
2741 #undef adj_init_size
2743 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
2744 #ifdef CONFIG_HIGHMEM
2748 K(nr_free_pages()), K(physpages
),
2749 codesize
/ SZ_1K
, datasize
/ SZ_1K
, rosize
/ SZ_1K
,
2750 (init_data_size
+ init_code_size
) / SZ_1K
, bss_size
/ SZ_1K
,
2751 K(physpages
- totalram_pages() - totalcma_pages
),
2753 #ifdef CONFIG_HIGHMEM
2754 , K(totalhigh_pages())
2760 * Set up kernel memory allocators
2762 void __init
mm_core_init(void)
2764 /* Initializations relying on SMP setup */
2765 build_all_zonelists(NULL
);
2766 page_alloc_init_cpuhp();
2769 * page_ext requires contiguous pages,
2770 * bigger than MAX_ORDER unless SPARSEMEM.
2772 page_ext_init_flatmem();
2773 mem_debugging_and_hardening_init();
2774 kfence_alloc_pool_and_metadata();
2776 kmsan_init_shadow();
2777 stack_depot_early_init();
2779 mem_init_print_info();
2782 * page_owner must be initialized after buddy is ready, and also after
2783 * slab is ready so that stack_depot_init() works properly
2785 page_ext_init_flatmem_late();
2787 ptlock_cache_init();
2788 pgtable_cache_init();
2789 debug_objects_mem_init();
2791 /* If no deferred init page_ext now, as vmap is fully initialized */
2792 if (!deferred_struct_pages
)
2794 /* Should be run before the first non-init thread is created */
2796 /* Should be run after espfix64 is set up. */
2798 kmsan_init_runtime();