1 // SPDX-License-Identifier: GPL-2.0
3 * sparse memory mappings.
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/memblock.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/bootmem_info.h>
22 * Permanent SPARSEMEM data:
24 * 1) mem_section - memory sections, mem_map's for valid memory
26 #ifdef CONFIG_SPARSEMEM_EXTREME
27 struct mem_section
**mem_section
;
29 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
30 ____cacheline_internodealigned_in_smp
;
32 EXPORT_SYMBOL(mem_section
);
34 #ifdef NODE_NOT_IN_PAGE_FLAGS
36 * If we did not store the node number in the page then we have to
37 * do a lookup in the section_to_node_table in order to find which
38 * node the page belongs to.
40 #if MAX_NUMNODES <= 256
41 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
43 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
46 int page_to_nid(const struct page
*page
)
48 return section_to_node_table
[page_to_section(page
)];
50 EXPORT_SYMBOL(page_to_nid
);
52 static void set_section_nid(unsigned long section_nr
, int nid
)
54 section_to_node_table
[section_nr
] = nid
;
56 #else /* !NODE_NOT_IN_PAGE_FLAGS */
57 static inline void set_section_nid(unsigned long section_nr
, int nid
)
62 #ifdef CONFIG_SPARSEMEM_EXTREME
63 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
65 struct mem_section
*section
= NULL
;
66 unsigned long array_size
= SECTIONS_PER_ROOT
*
67 sizeof(struct mem_section
);
69 if (slab_is_available()) {
70 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
72 section
= memblock_alloc_node(array_size
, SMP_CACHE_BYTES
,
75 panic("%s: Failed to allocate %lu bytes nid=%d\n",
76 __func__
, array_size
, nid
);
82 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
84 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
85 struct mem_section
*section
;
88 * An existing section is possible in the sub-section hotplug
89 * case. First hot-add instantiates, follow-on hot-add reuses
90 * the existing section.
92 * The mem_hotplug_lock resolves the apparent race below.
94 if (mem_section
[root
])
97 section
= sparse_index_alloc(nid
);
101 mem_section
[root
] = section
;
105 #else /* !SPARSEMEM_EXTREME */
106 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
113 * During early boot, before section_mem_map is used for an actual
114 * mem_map, we use section_mem_map to store the section's NUMA
115 * node. This keeps us from having to use another data structure. The
116 * node information is cleared just before we store the real mem_map.
118 static inline unsigned long sparse_encode_early_nid(int nid
)
120 return ((unsigned long)nid
<< SECTION_NID_SHIFT
);
123 static inline int sparse_early_nid(struct mem_section
*section
)
125 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
128 /* Validate the physical addressing limitations of the model */
129 static void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
130 unsigned long *end_pfn
)
132 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
135 * Sanity checks - do not allow an architecture to pass
136 * in larger pfns than the maximum scope of sparsemem:
138 if (*start_pfn
> max_sparsemem_pfn
) {
139 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
141 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
143 *start_pfn
= max_sparsemem_pfn
;
144 *end_pfn
= max_sparsemem_pfn
;
145 } else if (*end_pfn
> max_sparsemem_pfn
) {
146 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
148 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
150 *end_pfn
= max_sparsemem_pfn
;
155 * There are a number of times that we loop over NR_MEM_SECTIONS,
156 * looking for section_present() on each. But, when we have very
157 * large physical address spaces, NR_MEM_SECTIONS can also be
158 * very large which makes the loops quite long.
160 * Keeping track of this gives us an easy way to break out of
163 unsigned long __highest_present_section_nr
;
164 static void __section_mark_present(struct mem_section
*ms
,
165 unsigned long section_nr
)
167 if (section_nr
> __highest_present_section_nr
)
168 __highest_present_section_nr
= section_nr
;
170 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
173 #define for_each_present_section_nr(start, section_nr) \
174 for (section_nr = next_present_section_nr(start-1); \
176 section_nr = next_present_section_nr(section_nr))
178 static inline unsigned long first_present_section_nr(void)
180 return next_present_section_nr(-1);
183 #ifdef CONFIG_SPARSEMEM_VMEMMAP
184 static void subsection_mask_set(unsigned long *map
, unsigned long pfn
,
185 unsigned long nr_pages
)
187 int idx
= subsection_map_index(pfn
);
188 int end
= subsection_map_index(pfn
+ nr_pages
- 1);
190 bitmap_set(map
, idx
, end
- idx
+ 1);
193 void __init
subsection_map_init(unsigned long pfn
, unsigned long nr_pages
)
195 int end_sec
= pfn_to_section_nr(pfn
+ nr_pages
- 1);
196 unsigned long nr
, start_sec
= pfn_to_section_nr(pfn
);
201 for (nr
= start_sec
; nr
<= end_sec
; nr
++) {
202 struct mem_section
*ms
;
205 pfns
= min(nr_pages
, PAGES_PER_SECTION
206 - (pfn
& ~PAGE_SECTION_MASK
));
207 ms
= __nr_to_section(nr
);
208 subsection_mask_set(ms
->usage
->subsection_map
, pfn
, pfns
);
210 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__
, nr
,
211 pfns
, subsection_map_index(pfn
),
212 subsection_map_index(pfn
+ pfns
- 1));
219 void __init
subsection_map_init(unsigned long pfn
, unsigned long nr_pages
)
224 /* Record a memory area against a node. */
225 static void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
229 #ifdef CONFIG_SPARSEMEM_EXTREME
230 if (unlikely(!mem_section
)) {
231 unsigned long size
, align
;
233 size
= sizeof(struct mem_section
*) * NR_SECTION_ROOTS
;
234 align
= 1 << (INTERNODE_CACHE_SHIFT
);
235 mem_section
= memblock_alloc(size
, align
);
237 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
238 __func__
, size
, align
);
242 start
&= PAGE_SECTION_MASK
;
243 mminit_validate_memmodel_limits(&start
, &end
);
244 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
245 unsigned long section
= pfn_to_section_nr(pfn
);
246 struct mem_section
*ms
;
248 sparse_index_init(section
, nid
);
249 set_section_nid(section
, nid
);
251 ms
= __nr_to_section(section
);
252 if (!ms
->section_mem_map
) {
253 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
255 __section_mark_present(ms
, section
);
261 * Mark all memblocks as present using memory_present().
262 * This is a convenience function that is useful to mark all of the systems
263 * memory as present during initialization.
265 static void __init
memblocks_present(void)
267 unsigned long start
, end
;
270 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start
, &end
, &nid
)
271 memory_present(nid
, start
, end
);
275 * Subtle, we encode the real pfn into the mem_map such that
276 * the identity pfn - section_mem_map will return the actual
277 * physical page frame number.
279 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
281 unsigned long coded_mem_map
=
282 (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
283 BUILD_BUG_ON(SECTION_MAP_LAST_BIT
> PFN_SECTION_SHIFT
);
284 BUG_ON(coded_mem_map
& ~SECTION_MAP_MASK
);
285 return coded_mem_map
;
288 #ifdef CONFIG_MEMORY_HOTPLUG
290 * Decode mem_map from the coded memmap
292 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
294 /* mask off the extra low bits of information */
295 coded_mem_map
&= SECTION_MAP_MASK
;
296 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
298 #endif /* CONFIG_MEMORY_HOTPLUG */
300 static void __meminit
sparse_init_one_section(struct mem_section
*ms
,
301 unsigned long pnum
, struct page
*mem_map
,
302 struct mem_section_usage
*usage
, unsigned long flags
)
304 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
305 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
)
306 | SECTION_HAS_MEM_MAP
| flags
;
310 static unsigned long usemap_size(void)
312 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
315 size_t mem_section_usage_size(void)
317 return sizeof(struct mem_section_usage
) + usemap_size();
320 #ifdef CONFIG_MEMORY_HOTREMOVE
321 static inline phys_addr_t
pgdat_to_phys(struct pglist_data
*pgdat
)
324 VM_BUG_ON(pgdat
!= &contig_page_data
);
325 return __pa_symbol(&contig_page_data
);
331 static struct mem_section_usage
* __init
332 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
335 struct mem_section_usage
*usage
;
336 unsigned long goal
, limit
;
339 * A page may contain usemaps for other sections preventing the
340 * page being freed and making a section unremovable while
341 * other sections referencing the usemap remain active. Similarly,
342 * a pgdat can prevent a section being removed. If section A
343 * contains a pgdat and section B contains the usemap, both
344 * sections become inter-dependent. This allocates usemaps
345 * from the same section as the pgdat where possible to avoid
348 goal
= pgdat_to_phys(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
349 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
350 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
352 usage
= memblock_alloc_try_nid(size
, SMP_CACHE_BYTES
, goal
, limit
, nid
);
353 if (!usage
&& limit
) {
360 static void __init
check_usemap_section_nr(int nid
,
361 struct mem_section_usage
*usage
)
363 unsigned long usemap_snr
, pgdat_snr
;
364 static unsigned long old_usemap_snr
;
365 static unsigned long old_pgdat_snr
;
366 struct pglist_data
*pgdat
= NODE_DATA(nid
);
370 if (!old_usemap_snr
) {
371 old_usemap_snr
= NR_MEM_SECTIONS
;
372 old_pgdat_snr
= NR_MEM_SECTIONS
;
375 usemap_snr
= pfn_to_section_nr(__pa(usage
) >> PAGE_SHIFT
);
376 pgdat_snr
= pfn_to_section_nr(pgdat_to_phys(pgdat
) >> PAGE_SHIFT
);
377 if (usemap_snr
== pgdat_snr
)
380 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
381 /* skip redundant message */
384 old_usemap_snr
= usemap_snr
;
385 old_pgdat_snr
= pgdat_snr
;
387 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
388 if (usemap_nid
!= nid
) {
389 pr_info("node %d must be removed before remove section %ld\n",
394 * There is a circular dependency.
395 * Some platforms allow un-removable section because they will just
396 * gather other removable sections for dynamic partitioning.
397 * Just notify un-removable section's number here.
399 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
400 usemap_snr
, pgdat_snr
, nid
);
403 static struct mem_section_usage
* __init
404 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
407 return memblock_alloc_node(size
, SMP_CACHE_BYTES
, pgdat
->node_id
);
410 static void __init
check_usemap_section_nr(int nid
,
411 struct mem_section_usage
*usage
)
414 #endif /* CONFIG_MEMORY_HOTREMOVE */
416 #ifdef CONFIG_SPARSEMEM_VMEMMAP
417 static unsigned long __init
section_map_size(void)
419 return ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
, PMD_SIZE
);
423 static unsigned long __init
section_map_size(void)
425 return PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
428 struct page __init
*__populate_section_memmap(unsigned long pfn
,
429 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
430 struct dev_pagemap
*pgmap
)
432 unsigned long size
= section_map_size();
433 struct page
*map
= sparse_buffer_alloc(size
);
434 phys_addr_t addr
= __pa(MAX_DMA_ADDRESS
);
439 map
= memmap_alloc(size
, size
, addr
, nid
, false);
441 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
442 __func__
, size
, PAGE_SIZE
, nid
, &addr
);
446 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
448 static void *sparsemap_buf __meminitdata
;
449 static void *sparsemap_buf_end __meminitdata
;
451 static inline void __meminit
sparse_buffer_free(unsigned long size
)
453 WARN_ON(!sparsemap_buf
|| size
== 0);
454 memblock_free(sparsemap_buf
, size
);
457 static void __init
sparse_buffer_init(unsigned long size
, int nid
)
459 phys_addr_t addr
= __pa(MAX_DMA_ADDRESS
);
460 WARN_ON(sparsemap_buf
); /* forgot to call sparse_buffer_fini()? */
462 * Pre-allocated buffer is mainly used by __populate_section_memmap
463 * and we want it to be properly aligned to the section size - this is
464 * especially the case for VMEMMAP which maps memmap to PMDs
466 sparsemap_buf
= memmap_alloc(size
, section_map_size(), addr
, nid
, true);
467 sparsemap_buf_end
= sparsemap_buf
+ size
;
470 static void __init
sparse_buffer_fini(void)
472 unsigned long size
= sparsemap_buf_end
- sparsemap_buf
;
474 if (sparsemap_buf
&& size
> 0)
475 sparse_buffer_free(size
);
476 sparsemap_buf
= NULL
;
479 void * __meminit
sparse_buffer_alloc(unsigned long size
)
484 ptr
= (void *) roundup((unsigned long)sparsemap_buf
, size
);
485 if (ptr
+ size
> sparsemap_buf_end
)
488 /* Free redundant aligned space */
489 if ((unsigned long)(ptr
- sparsemap_buf
) > 0)
490 sparse_buffer_free((unsigned long)(ptr
- sparsemap_buf
));
491 sparsemap_buf
= ptr
+ size
;
497 void __weak __meminit
vmemmap_populate_print_last(void)
502 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
503 * And number of present sections in this node is map_count.
505 static void __init
sparse_init_nid(int nid
, unsigned long pnum_begin
,
506 unsigned long pnum_end
,
507 unsigned long map_count
)
509 struct mem_section_usage
*usage
;
513 usage
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nid
),
514 mem_section_usage_size() * map_count
);
516 pr_err("%s: node[%d] usemap allocation failed", __func__
, nid
);
519 sparse_buffer_init(map_count
* section_map_size(), nid
);
520 for_each_present_section_nr(pnum_begin
, pnum
) {
521 unsigned long pfn
= section_nr_to_pfn(pnum
);
523 if (pnum
>= pnum_end
)
526 map
= __populate_section_memmap(pfn
, PAGES_PER_SECTION
,
529 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
532 sparse_buffer_fini();
535 check_usemap_section_nr(nid
, usage
);
536 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
, usage
,
538 usage
= (void *) usage
+ mem_section_usage_size();
540 sparse_buffer_fini();
543 /* We failed to allocate, mark all the following pnums as not present */
544 for_each_present_section_nr(pnum_begin
, pnum
) {
545 struct mem_section
*ms
;
547 if (pnum
>= pnum_end
)
549 ms
= __nr_to_section(pnum
);
550 ms
->section_mem_map
= 0;
555 * Allocate the accumulated non-linear sections, allocate a mem_map
556 * for each and record the physical to section mapping.
558 void __init
sparse_init(void)
560 unsigned long pnum_end
, pnum_begin
, map_count
= 1;
565 pnum_begin
= first_present_section_nr();
566 nid_begin
= sparse_early_nid(__nr_to_section(pnum_begin
));
568 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
569 set_pageblock_order();
571 for_each_present_section_nr(pnum_begin
+ 1, pnum_end
) {
572 int nid
= sparse_early_nid(__nr_to_section(pnum_end
));
574 if (nid
== nid_begin
) {
578 /* Init node with sections in range [pnum_begin, pnum_end) */
579 sparse_init_nid(nid_begin
, pnum_begin
, pnum_end
, map_count
);
581 pnum_begin
= pnum_end
;
584 /* cover the last node */
585 sparse_init_nid(nid_begin
, pnum_begin
, pnum_end
, map_count
);
586 vmemmap_populate_print_last();
589 #ifdef CONFIG_MEMORY_HOTPLUG
591 /* Mark all memory sections within the pfn range as online */
592 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
596 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
597 unsigned long section_nr
= pfn_to_section_nr(pfn
);
598 struct mem_section
*ms
;
600 /* onlining code should never touch invalid ranges */
601 if (WARN_ON(!valid_section_nr(section_nr
)))
604 ms
= __nr_to_section(section_nr
);
605 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
609 /* Mark all memory sections within the pfn range as offline */
610 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
614 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
615 unsigned long section_nr
= pfn_to_section_nr(pfn
);
616 struct mem_section
*ms
;
619 * TODO this needs some double checking. Offlining code makes
620 * sure to check pfn_valid but those checks might be just bogus
622 if (WARN_ON(!valid_section_nr(section_nr
)))
625 ms
= __nr_to_section(section_nr
);
626 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
630 #ifdef CONFIG_SPARSEMEM_VMEMMAP
631 static struct page
* __meminit
populate_section_memmap(unsigned long pfn
,
632 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
633 struct dev_pagemap
*pgmap
)
635 return __populate_section_memmap(pfn
, nr_pages
, nid
, altmap
, pgmap
);
638 static void depopulate_section_memmap(unsigned long pfn
, unsigned long nr_pages
,
639 struct vmem_altmap
*altmap
)
641 unsigned long start
= (unsigned long) pfn_to_page(pfn
);
642 unsigned long end
= start
+ nr_pages
* sizeof(struct page
);
644 vmemmap_free(start
, end
, altmap
);
646 static void free_map_bootmem(struct page
*memmap
)
648 unsigned long start
= (unsigned long)memmap
;
649 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
651 vmemmap_free(start
, end
, NULL
);
654 static int clear_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
656 DECLARE_BITMAP(map
, SUBSECTIONS_PER_SECTION
) = { 0 };
657 DECLARE_BITMAP(tmp
, SUBSECTIONS_PER_SECTION
) = { 0 };
658 struct mem_section
*ms
= __pfn_to_section(pfn
);
659 unsigned long *subsection_map
= ms
->usage
660 ? &ms
->usage
->subsection_map
[0] : NULL
;
662 subsection_mask_set(map
, pfn
, nr_pages
);
664 bitmap_and(tmp
, map
, subsection_map
, SUBSECTIONS_PER_SECTION
);
666 if (WARN(!subsection_map
|| !bitmap_equal(tmp
, map
, SUBSECTIONS_PER_SECTION
),
667 "section already deactivated (%#lx + %ld)\n",
671 bitmap_xor(subsection_map
, map
, subsection_map
, SUBSECTIONS_PER_SECTION
);
675 static bool is_subsection_map_empty(struct mem_section
*ms
)
677 return bitmap_empty(&ms
->usage
->subsection_map
[0],
678 SUBSECTIONS_PER_SECTION
);
681 static int fill_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
683 struct mem_section
*ms
= __pfn_to_section(pfn
);
684 DECLARE_BITMAP(map
, SUBSECTIONS_PER_SECTION
) = { 0 };
685 unsigned long *subsection_map
;
688 subsection_mask_set(map
, pfn
, nr_pages
);
690 subsection_map
= &ms
->usage
->subsection_map
[0];
692 if (bitmap_empty(map
, SUBSECTIONS_PER_SECTION
))
694 else if (bitmap_intersects(map
, subsection_map
, SUBSECTIONS_PER_SECTION
))
697 bitmap_or(subsection_map
, map
, subsection_map
,
698 SUBSECTIONS_PER_SECTION
);
703 static struct page
* __meminit
populate_section_memmap(unsigned long pfn
,
704 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
705 struct dev_pagemap
*pgmap
)
707 return kvmalloc_node(array_size(sizeof(struct page
),
708 PAGES_PER_SECTION
), GFP_KERNEL
, nid
);
711 static void depopulate_section_memmap(unsigned long pfn
, unsigned long nr_pages
,
712 struct vmem_altmap
*altmap
)
714 kvfree(pfn_to_page(pfn
));
717 static void free_map_bootmem(struct page
*memmap
)
719 unsigned long maps_section_nr
, removing_section_nr
, i
;
720 unsigned long magic
, nr_pages
;
721 struct page
*page
= virt_to_page(memmap
);
723 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
726 for (i
= 0; i
< nr_pages
; i
++, page
++) {
729 BUG_ON(magic
== NODE_INFO
);
731 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
732 removing_section_nr
= page_private(page
);
735 * When this function is called, the removing section is
736 * logical offlined state. This means all pages are isolated
737 * from page allocator. If removing section's memmap is placed
738 * on the same section, it must not be freed.
739 * If it is freed, page allocator may allocate it which will
740 * be removed physically soon.
742 if (maps_section_nr
!= removing_section_nr
)
743 put_page_bootmem(page
);
747 static int clear_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
752 static bool is_subsection_map_empty(struct mem_section
*ms
)
757 static int fill_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
761 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
764 * To deactivate a memory region, there are 3 cases to handle across
765 * two configurations (SPARSEMEM_VMEMMAP={y,n}):
767 * 1. deactivation of a partial hot-added section (only possible in
768 * the SPARSEMEM_VMEMMAP=y case).
769 * a) section was present at memory init.
770 * b) section was hot-added post memory init.
771 * 2. deactivation of a complete hot-added section.
772 * 3. deactivation of a complete section from memory init.
774 * For 1, when subsection_map does not empty we will not be freeing the
775 * usage map, but still need to free the vmemmap range.
777 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified
779 static void section_deactivate(unsigned long pfn
, unsigned long nr_pages
,
780 struct vmem_altmap
*altmap
)
782 struct mem_section
*ms
= __pfn_to_section(pfn
);
783 bool section_is_early
= early_section(ms
);
784 struct page
*memmap
= NULL
;
787 if (clear_subsection_map(pfn
, nr_pages
))
790 empty
= is_subsection_map_empty(ms
);
792 unsigned long section_nr
= pfn_to_section_nr(pfn
);
795 * When removing an early section, the usage map is kept (as the
796 * usage maps of other sections fall into the same page). It
797 * will be re-used when re-adding the section - which is then no
798 * longer an early section. If the usage map is PageReserved, it
799 * was allocated during boot.
801 if (!PageReserved(virt_to_page(ms
->usage
))) {
805 memmap
= sparse_decode_mem_map(ms
->section_mem_map
, section_nr
);
807 * Mark the section invalid so that valid_section()
808 * return false. This prevents code from dereferencing
811 ms
->section_mem_map
&= ~SECTION_HAS_MEM_MAP
;
815 * The memmap of early sections is always fully populated. See
816 * section_activate() and pfn_valid() .
818 if (!section_is_early
)
819 depopulate_section_memmap(pfn
, nr_pages
, altmap
);
821 free_map_bootmem(memmap
);
824 ms
->section_mem_map
= (unsigned long)NULL
;
827 static struct page
* __meminit
section_activate(int nid
, unsigned long pfn
,
828 unsigned long nr_pages
, struct vmem_altmap
*altmap
,
829 struct dev_pagemap
*pgmap
)
831 struct mem_section
*ms
= __pfn_to_section(pfn
);
832 struct mem_section_usage
*usage
= NULL
;
837 usage
= kzalloc(mem_section_usage_size(), GFP_KERNEL
);
839 return ERR_PTR(-ENOMEM
);
843 rc
= fill_subsection_map(pfn
, nr_pages
);
852 * The early init code does not consider partially populated
853 * initial sections, it simply assumes that memory will never be
854 * referenced. If we hot-add memory into such a section then we
855 * do not need to populate the memmap and can simply reuse what
858 if (nr_pages
< PAGES_PER_SECTION
&& early_section(ms
))
859 return pfn_to_page(pfn
);
861 memmap
= populate_section_memmap(pfn
, nr_pages
, nid
, altmap
, pgmap
);
863 section_deactivate(pfn
, nr_pages
, altmap
);
864 return ERR_PTR(-ENOMEM
);
871 * sparse_add_section - add a memory section, or populate an existing one
872 * @nid: The node to add section on
873 * @start_pfn: start pfn of the memory range
874 * @nr_pages: number of pfns to add in the section
875 * @altmap: alternate pfns to allocate the memmap backing store
876 * @pgmap: alternate compound page geometry for devmap mappings
878 * This is only intended for hotplug.
880 * Note that only VMEMMAP supports sub-section aligned hotplug,
881 * the proper alignment and size are gated by check_pfn_span().
886 * * -EEXIST - Section has been present.
887 * * -ENOMEM - Out of memory.
889 int __meminit
sparse_add_section(int nid
, unsigned long start_pfn
,
890 unsigned long nr_pages
, struct vmem_altmap
*altmap
,
891 struct dev_pagemap
*pgmap
)
893 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
894 struct mem_section
*ms
;
898 ret
= sparse_index_init(section_nr
, nid
);
902 memmap
= section_activate(nid
, start_pfn
, nr_pages
, altmap
, pgmap
);
904 return PTR_ERR(memmap
);
907 * Poison uninitialized struct pages in order to catch invalid flags
910 page_init_poison(memmap
, sizeof(struct page
) * nr_pages
);
912 ms
= __nr_to_section(section_nr
);
913 set_section_nid(section_nr
, nid
);
914 __section_mark_present(ms
, section_nr
);
916 /* Align memmap to section boundary in the subsection case */
917 if (section_nr_to_pfn(section_nr
) != start_pfn
)
918 memmap
= pfn_to_page(section_nr_to_pfn(section_nr
));
919 sparse_init_one_section(ms
, section_nr
, memmap
, ms
->usage
, 0);
924 void sparse_remove_section(unsigned long pfn
, unsigned long nr_pages
,
925 struct vmem_altmap
*altmap
)
927 struct mem_section
*ms
= __pfn_to_section(pfn
);
929 if (WARN_ON_ONCE(!valid_section(ms
)))
932 section_deactivate(pfn
, nr_pages
, altmap
);
934 #endif /* CONFIG_MEMORY_HOTPLUG */