1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Contiguous Memory Allocator
5 * Copyright (c) 2010-2011 by Samsung Electronics.
6 * Copyright IBM Corporation, 2013
7 * Copyright LG Electronics Inc., 2014
9 * Marek Szyprowski <m.szyprowski@samsung.com>
10 * Michal Nazarewicz <mina86@mina86.com>
11 * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
12 * Joonsoo Kim <iamjoonsoo.kim@lge.com>
15 #define pr_fmt(fmt) "cma: " fmt
17 #define CREATE_TRACE_POINTS
19 #include <linux/memblock.h>
20 #include <linux/err.h>
22 #include <linux/sizes.h>
23 #include <linux/slab.h>
24 #include <linux/log2.h>
25 #include <linux/cma.h>
26 #include <linux/highmem.h>
28 #include <linux/kmemleak.h>
29 #include <trace/events/cma.h>
34 struct cma cma_areas
[MAX_CMA_AREAS
];
35 unsigned cma_area_count
;
36 static DEFINE_MUTEX(cma_mutex
);
38 phys_addr_t
cma_get_base(const struct cma
*cma
)
40 return PFN_PHYS(cma
->base_pfn
);
43 unsigned long cma_get_size(const struct cma
*cma
)
45 return cma
->count
<< PAGE_SHIFT
;
48 const char *cma_get_name(const struct cma
*cma
)
53 static unsigned long cma_bitmap_aligned_mask(const struct cma
*cma
,
54 unsigned int align_order
)
56 if (align_order
<= cma
->order_per_bit
)
58 return (1UL << (align_order
- cma
->order_per_bit
)) - 1;
62 * Find the offset of the base PFN from the specified align_order.
63 * The value returned is represented in order_per_bits.
65 static unsigned long cma_bitmap_aligned_offset(const struct cma
*cma
,
66 unsigned int align_order
)
68 return (cma
->base_pfn
& ((1UL << align_order
) - 1))
69 >> cma
->order_per_bit
;
72 static unsigned long cma_bitmap_pages_to_bits(const struct cma
*cma
,
75 return ALIGN(pages
, 1UL << cma
->order_per_bit
) >> cma
->order_per_bit
;
78 static void cma_clear_bitmap(struct cma
*cma
, unsigned long pfn
,
81 unsigned long bitmap_no
, bitmap_count
;
84 bitmap_no
= (pfn
- cma
->base_pfn
) >> cma
->order_per_bit
;
85 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
87 spin_lock_irqsave(&cma
->lock
, flags
);
88 bitmap_clear(cma
->bitmap
, bitmap_no
, bitmap_count
);
89 spin_unlock_irqrestore(&cma
->lock
, flags
);
92 static void __init
cma_activate_area(struct cma
*cma
)
94 unsigned long base_pfn
= cma
->base_pfn
, pfn
;
97 cma
->bitmap
= bitmap_zalloc(cma_bitmap_maxno(cma
), GFP_KERNEL
);
102 * alloc_contig_range() requires the pfn range specified to be in the
103 * same zone. Simplify by forcing the entire CMA resv range to be in the
106 WARN_ON_ONCE(!pfn_valid(base_pfn
));
107 zone
= page_zone(pfn_to_page(base_pfn
));
108 for (pfn
= base_pfn
+ 1; pfn
< base_pfn
+ cma
->count
; pfn
++) {
109 WARN_ON_ONCE(!pfn_valid(pfn
));
110 if (page_zone(pfn_to_page(pfn
)) != zone
)
114 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
;
115 pfn
+= pageblock_nr_pages
)
116 init_cma_reserved_pageblock(pfn_to_page(pfn
));
118 spin_lock_init(&cma
->lock
);
120 #ifdef CONFIG_CMA_DEBUGFS
121 INIT_HLIST_HEAD(&cma
->mem_head
);
122 spin_lock_init(&cma
->mem_head_lock
);
128 bitmap_free(cma
->bitmap
);
130 /* Expose all pages to the buddy, they are useless for CMA. */
131 if (!cma
->reserve_pages_on_error
) {
132 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
; pfn
++)
133 free_reserved_page(pfn_to_page(pfn
));
135 totalcma_pages
-= cma
->count
;
137 pr_err("CMA area %s could not be activated\n", cma
->name
);
141 static int __init
cma_init_reserved_areas(void)
145 for (i
= 0; i
< cma_area_count
; i
++)
146 cma_activate_area(&cma_areas
[i
]);
150 core_initcall(cma_init_reserved_areas
);
152 void __init
cma_reserve_pages_on_error(struct cma
*cma
)
154 cma
->reserve_pages_on_error
= true;
158 * cma_init_reserved_mem() - create custom contiguous area from reserved memory
159 * @base: Base address of the reserved area
160 * @size: Size of the reserved area (in bytes),
161 * @order_per_bit: Order of pages represented by one bit on bitmap.
162 * @name: The name of the area. If this parameter is NULL, the name of
163 * the area will be set to "cmaN", where N is a running counter of
165 * @res_cma: Pointer to store the created cma region.
167 * This function creates custom contiguous area from already reserved memory.
169 int __init
cma_init_reserved_mem(phys_addr_t base
, phys_addr_t size
,
170 unsigned int order_per_bit
,
172 struct cma
**res_cma
)
177 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
178 pr_err("Not enough slots for CMA reserved regions!\n");
182 if (!size
|| !memblock_is_region_reserved(base
, size
))
185 /* alignment should be aligned with order_per_bit */
186 if (!IS_ALIGNED(CMA_MIN_ALIGNMENT_PAGES
, 1 << order_per_bit
))
189 /* ensure minimal alignment required by mm core */
190 if (!IS_ALIGNED(base
| size
, CMA_MIN_ALIGNMENT_BYTES
))
194 * Each reserved area must be initialised later, when more kernel
195 * subsystems (like slab allocator) are available.
197 cma
= &cma_areas
[cma_area_count
];
200 snprintf(cma
->name
, CMA_MAX_NAME
, name
);
202 snprintf(cma
->name
, CMA_MAX_NAME
, "cma%d\n", cma_area_count
);
204 cma
->base_pfn
= PFN_DOWN(base
);
205 cma
->count
= size
>> PAGE_SHIFT
;
206 cma
->order_per_bit
= order_per_bit
;
209 totalcma_pages
+= (size
/ PAGE_SIZE
);
215 * cma_declare_contiguous_nid() - reserve custom contiguous area
216 * @base: Base address of the reserved area optional, use 0 for any
217 * @size: Size of the reserved area (in bytes),
218 * @limit: End address of the reserved memory (optional, 0 for any).
219 * @alignment: Alignment for the CMA area, should be power of 2 or zero
220 * @order_per_bit: Order of pages represented by one bit on bitmap.
221 * @fixed: hint about where to place the reserved area
222 * @name: The name of the area. See function cma_init_reserved_mem()
223 * @res_cma: Pointer to store the created cma region.
224 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
226 * This function reserves memory from early allocator. It should be
227 * called by arch specific code once the early allocator (memblock or bootmem)
228 * has been activated and all other subsystems have already allocated/reserved
229 * memory. This function allows to create custom reserved areas.
231 * If @fixed is true, reserve contiguous area at exactly @base. If false,
232 * reserve in range from @base to @limit.
234 int __init
cma_declare_contiguous_nid(phys_addr_t base
,
235 phys_addr_t size
, phys_addr_t limit
,
236 phys_addr_t alignment
, unsigned int order_per_bit
,
237 bool fixed
, const char *name
, struct cma
**res_cma
,
240 phys_addr_t memblock_end
= memblock_end_of_DRAM();
241 phys_addr_t highmem_start
;
245 * We can't use __pa(high_memory) directly, since high_memory
246 * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly)
247 * complain. Find the boundary by adding one to the last valid
250 highmem_start
= __pa(high_memory
- 1) + 1;
251 pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
252 __func__
, &size
, &base
, &limit
, &alignment
);
254 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
255 pr_err("Not enough slots for CMA reserved regions!\n");
262 if (alignment
&& !is_power_of_2(alignment
))
265 if (!IS_ENABLED(CONFIG_NUMA
))
268 /* Sanitise input arguments. */
269 alignment
= max_t(phys_addr_t
, alignment
, CMA_MIN_ALIGNMENT_BYTES
);
270 if (fixed
&& base
& (alignment
- 1)) {
272 pr_err("Region at %pa must be aligned to %pa bytes\n",
276 base
= ALIGN(base
, alignment
);
277 size
= ALIGN(size
, alignment
);
278 limit
&= ~(alignment
- 1);
283 /* size should be aligned with order_per_bit */
284 if (!IS_ALIGNED(size
>> PAGE_SHIFT
, 1 << order_per_bit
))
288 * If allocating at a fixed base the request region must not cross the
289 * low/high memory boundary.
291 if (fixed
&& base
< highmem_start
&& base
+ size
> highmem_start
) {
293 pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
294 &base
, &highmem_start
);
299 * If the limit is unspecified or above the memblock end, its effective
300 * value will be the memblock end. Set it explicitly to simplify further
303 if (limit
== 0 || limit
> memblock_end
)
304 limit
= memblock_end
;
306 if (base
+ size
> limit
) {
308 pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n",
309 &size
, &base
, &limit
);
315 if (memblock_is_region_reserved(base
, size
) ||
316 memblock_reserve(base
, size
) < 0) {
321 phys_addr_t addr
= 0;
324 * If there is enough memory, try a bottom-up allocation first.
325 * It will place the new cma area close to the start of the node
326 * and guarantee that the compaction is moving pages out of the
327 * cma area and not into it.
328 * Avoid using first 4GB to not interfere with constrained zones
331 #ifdef CONFIG_PHYS_ADDR_T_64BIT
332 if (!memblock_bottom_up() && memblock_end
>= SZ_4G
+ size
) {
333 memblock_set_bottom_up(true);
334 addr
= memblock_alloc_range_nid(size
, alignment
, SZ_4G
,
336 memblock_set_bottom_up(false);
341 * All pages in the reserved area must come from the same zone.
342 * If the requested region crosses the low/high memory boundary,
343 * try allocating from high memory first and fall back to low
344 * memory in case of failure.
346 if (!addr
&& base
< highmem_start
&& limit
> highmem_start
) {
347 addr
= memblock_alloc_range_nid(size
, alignment
,
348 highmem_start
, limit
, nid
, true);
349 limit
= highmem_start
;
353 addr
= memblock_alloc_range_nid(size
, alignment
, base
,
362 * kmemleak scans/reads tracked objects for pointers to other
363 * objects but this address isn't mapped and accessible
365 kmemleak_ignore_phys(addr
);
369 ret
= cma_init_reserved_mem(base
, size
, order_per_bit
, name
, res_cma
);
373 pr_info("Reserved %ld MiB at %pa on node %d\n", (unsigned long)size
/ SZ_1M
,
378 memblock_phys_free(base
, size
);
380 pr_err("Failed to reserve %ld MiB on node %d\n", (unsigned long)size
/ SZ_1M
,
385 static void cma_debug_show_areas(struct cma
*cma
)
387 unsigned long next_zero_bit
, next_set_bit
, nr_zero
;
388 unsigned long start
= 0;
389 unsigned long nr_part
, nr_total
= 0;
390 unsigned long nbits
= cma_bitmap_maxno(cma
);
392 spin_lock_irq(&cma
->lock
);
393 pr_info("number of available pages: ");
395 next_zero_bit
= find_next_zero_bit(cma
->bitmap
, nbits
, start
);
396 if (next_zero_bit
>= nbits
)
398 next_set_bit
= find_next_bit(cma
->bitmap
, nbits
, next_zero_bit
);
399 nr_zero
= next_set_bit
- next_zero_bit
;
400 nr_part
= nr_zero
<< cma
->order_per_bit
;
401 pr_cont("%s%lu@%lu", nr_total
? "+" : "", nr_part
,
404 start
= next_zero_bit
+ nr_zero
;
406 pr_cont("=> %lu free of %lu total pages\n", nr_total
, cma
->count
);
407 spin_unlock_irq(&cma
->lock
);
411 * cma_alloc() - allocate pages from contiguous area
412 * @cma: Contiguous memory region for which the allocation is performed.
413 * @count: Requested number of pages.
414 * @align: Requested alignment of pages (in PAGE_SIZE order).
415 * @no_warn: Avoid printing message about failed allocation
417 * This function allocates part of contiguous memory on specific
418 * contiguous memory area.
420 struct page
*cma_alloc(struct cma
*cma
, unsigned long count
,
421 unsigned int align
, bool no_warn
)
423 unsigned long mask
, offset
;
424 unsigned long pfn
= -1;
425 unsigned long start
= 0;
426 unsigned long bitmap_maxno
, bitmap_no
, bitmap_count
;
428 struct page
*page
= NULL
;
430 const char *name
= cma
? cma
->name
: NULL
;
432 trace_cma_alloc_start(name
, count
, align
);
434 if (!cma
|| !cma
->count
|| !cma
->bitmap
)
437 pr_debug("%s(cma %p, name: %s, count %lu, align %d)\n", __func__
,
438 (void *)cma
, cma
->name
, count
, align
);
443 mask
= cma_bitmap_aligned_mask(cma
, align
);
444 offset
= cma_bitmap_aligned_offset(cma
, align
);
445 bitmap_maxno
= cma_bitmap_maxno(cma
);
446 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
448 if (bitmap_count
> bitmap_maxno
)
452 spin_lock_irq(&cma
->lock
);
453 bitmap_no
= bitmap_find_next_zero_area_off(cma
->bitmap
,
454 bitmap_maxno
, start
, bitmap_count
, mask
,
456 if (bitmap_no
>= bitmap_maxno
) {
457 spin_unlock_irq(&cma
->lock
);
460 bitmap_set(cma
->bitmap
, bitmap_no
, bitmap_count
);
462 * It's safe to drop the lock here. We've marked this region for
463 * our exclusive use. If the migration fails we will take the
464 * lock again and unmark it.
466 spin_unlock_irq(&cma
->lock
);
468 pfn
= cma
->base_pfn
+ (bitmap_no
<< cma
->order_per_bit
);
469 mutex_lock(&cma_mutex
);
470 ret
= alloc_contig_range(pfn
, pfn
+ count
, MIGRATE_CMA
,
471 GFP_KERNEL
| (no_warn
? __GFP_NOWARN
: 0));
472 mutex_unlock(&cma_mutex
);
474 page
= pfn_to_page(pfn
);
478 cma_clear_bitmap(cma
, pfn
, count
);
482 pr_debug("%s(): memory range at pfn 0x%lx %p is busy, retrying\n",
483 __func__
, pfn
, pfn_to_page(pfn
));
485 trace_cma_alloc_busy_retry(cma
->name
, pfn
, pfn_to_page(pfn
),
487 /* try again with a bit different memory target */
488 start
= bitmap_no
+ mask
+ 1;
492 * CMA can allocate multiple page blocks, which results in different
493 * blocks being marked with different tags. Reset the tags to ignore
497 for (i
= 0; i
< count
; i
++)
498 page_kasan_tag_reset(nth_page(page
, i
));
501 if (ret
&& !no_warn
) {
502 pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
503 __func__
, cma
->name
, count
, ret
);
504 cma_debug_show_areas(cma
);
507 pr_debug("%s(): returned %p\n", __func__
, page
);
508 trace_cma_alloc_finish(name
, pfn
, page
, count
, align
, ret
);
510 count_vm_event(CMA_ALLOC_SUCCESS
);
511 cma_sysfs_account_success_pages(cma
, count
);
513 count_vm_event(CMA_ALLOC_FAIL
);
514 cma_sysfs_account_fail_pages(cma
, count
);
520 bool cma_pages_valid(struct cma
*cma
, const struct page
*pages
,
528 pfn
= page_to_pfn(pages
);
530 if (pfn
< cma
->base_pfn
|| pfn
>= cma
->base_pfn
+ cma
->count
) {
531 pr_debug("%s(page %p, count %lu)\n", __func__
,
532 (void *)pages
, count
);
540 * cma_release() - release allocated pages
541 * @cma: Contiguous memory region for which the allocation is performed.
542 * @pages: Allocated pages.
543 * @count: Number of allocated pages.
545 * This function releases memory allocated by cma_alloc().
546 * It returns false when provided pages do not belong to contiguous area and
549 bool cma_release(struct cma
*cma
, const struct page
*pages
,
554 if (!cma_pages_valid(cma
, pages
, count
))
557 pr_debug("%s(page %p, count %lu)\n", __func__
, (void *)pages
, count
);
559 pfn
= page_to_pfn(pages
);
561 VM_BUG_ON(pfn
+ count
> cma
->base_pfn
+ cma
->count
);
563 free_contig_range(pfn
, count
);
564 cma_clear_bitmap(cma
, pfn
, count
);
565 cma_sysfs_account_release_pages(cma
, count
);
566 trace_cma_release(cma
->name
, pfn
, pages
, count
);
571 int cma_for_each_area(int (*it
)(struct cma
*cma
, void *data
), void *data
)
575 for (i
= 0; i
< cma_area_count
; i
++) {
576 int ret
= it(&cma_areas
[i
], data
);