1 // SPDX-License-Identifier: GPL-2.0+
3 * EFI application memory management
5 * Copyright (c) 2016 Alexander Graf
9 #include <efi_loader.h>
14 #include <linux/list_sort.h>
15 #include <linux/sizes.h>
17 DECLARE_GLOBAL_DATA_PTR
;
19 /* Magic number identifying memory allocated from pool */
20 #define EFI_ALLOC_POOL_MAGIC 0x1fe67ddf6491caa2
22 efi_uintn_t efi_memory_map_key
;
25 struct list_head link
;
26 struct efi_mem_desc desc
;
29 #define EFI_CARVE_NO_OVERLAP -1
30 #define EFI_CARVE_LOOP_AGAIN -2
31 #define EFI_CARVE_OVERLAPS_NONRAM -3
33 /* This list contains all memory map items */
36 #ifdef CONFIG_EFI_LOADER_BOUNCE_BUFFER
37 void *efi_bounce_buffer
;
41 * struct efi_pool_allocation - memory block allocated from pool
43 * @num_pages: number of pages allocated
45 * @data: allocated pool memory
47 * U-Boot services each UEFI AllocatePool() request as a separate
48 * (multiple) page allocation. We have to track the number of pages
49 * to be able to free the correct amount later.
51 * The checksum calculated in function checksum() is used in FreePool() to avoid
52 * freeing memory not allocated by AllocatePool() and duplicate freeing.
54 * EFI requires 8 byte alignment for pool allocations, so we can
55 * prepend each allocation with these header fields.
57 struct efi_pool_allocation
{
60 char data
[] __aligned(ARCH_DMA_MINALIGN
);
64 * checksum() - calculate checksum for memory allocated from pool
66 * @alloc: allocation header
67 * Return: checksum, always non-zero
69 static u64
checksum(struct efi_pool_allocation
*alloc
)
71 u64 addr
= (uintptr_t)alloc
;
72 u64 ret
= (addr
>> 32) ^ (addr
<< 32) ^ alloc
->num_pages
^
80 * Sorts the memory list from highest address to lowest address
82 * When allocating memory we should always start from the highest
83 * address chunk, so sort the memory list such that the first list
84 * iterator gets the highest address and goes lower from there.
86 static int efi_mem_cmp(void *priv
, struct list_head
*a
, struct list_head
*b
)
88 struct efi_mem_list
*mema
= list_entry(a
, struct efi_mem_list
, link
);
89 struct efi_mem_list
*memb
= list_entry(b
, struct efi_mem_list
, link
);
91 if (mema
->desc
.physical_start
== memb
->desc
.physical_start
)
93 else if (mema
->desc
.physical_start
< memb
->desc
.physical_start
)
99 static uint64_t desc_get_end(struct efi_mem_desc
*desc
)
101 return desc
->physical_start
+ (desc
->num_pages
<< EFI_PAGE_SHIFT
);
104 static void efi_mem_sort(void)
106 struct list_head
*lhandle
;
107 struct efi_mem_list
*prevmem
= NULL
;
108 bool merge_again
= true;
110 list_sort(NULL
, &efi_mem
, efi_mem_cmp
);
112 /* Now merge entries that can be merged */
113 while (merge_again
) {
115 list_for_each(lhandle
, &efi_mem
) {
116 struct efi_mem_list
*lmem
;
117 struct efi_mem_desc
*prev
= &prevmem
->desc
;
118 struct efi_mem_desc
*cur
;
121 lmem
= list_entry(lhandle
, struct efi_mem_list
, link
);
129 if ((desc_get_end(cur
) == prev
->physical_start
) &&
130 (prev
->type
== cur
->type
) &&
131 (prev
->attribute
== cur
->attribute
)) {
132 /* There is an existing map before, reuse it */
133 pages
= cur
->num_pages
;
134 prev
->num_pages
+= pages
;
135 prev
->physical_start
-= pages
<< EFI_PAGE_SHIFT
;
136 prev
->virtual_start
-= pages
<< EFI_PAGE_SHIFT
;
137 list_del(&lmem
->link
);
149 /** efi_mem_carve_out - unmap memory region
152 * @carve_desc: memory region to unmap
153 * @overlap_only_ram: the carved out region may only overlap RAM
154 * Return Value: the number of overlapping pages which have been
155 * removed from the map,
156 * EFI_CARVE_NO_OVERLAP, if the regions don't overlap,
157 * EFI_CARVE_OVERLAPS_NONRAM, if the carve and map overlap,
158 * and the map contains anything but free ram
159 * (only when overlap_only_ram is true),
160 * EFI_CARVE_LOOP_AGAIN, if the mapping list should be
161 * traversed again, as it has been altered.
163 * Unmaps all memory occupied by the carve_desc region from the list entry
166 * In case of EFI_CARVE_OVERLAPS_NONRAM it is the callers responsibility
167 * to re-add the already carved out pages to the mapping.
169 static s64
efi_mem_carve_out(struct efi_mem_list
*map
,
170 struct efi_mem_desc
*carve_desc
,
171 bool overlap_only_ram
)
173 struct efi_mem_list
*newmap
;
174 struct efi_mem_desc
*map_desc
= &map
->desc
;
175 uint64_t map_start
= map_desc
->physical_start
;
176 uint64_t map_end
= map_start
+ (map_desc
->num_pages
<< EFI_PAGE_SHIFT
);
177 uint64_t carve_start
= carve_desc
->physical_start
;
178 uint64_t carve_end
= carve_start
+
179 (carve_desc
->num_pages
<< EFI_PAGE_SHIFT
);
181 /* check whether we're overlapping */
182 if ((carve_end
<= map_start
) || (carve_start
>= map_end
))
183 return EFI_CARVE_NO_OVERLAP
;
185 /* We're overlapping with non-RAM, warn the caller if desired */
186 if (overlap_only_ram
&& (map_desc
->type
!= EFI_CONVENTIONAL_MEMORY
))
187 return EFI_CARVE_OVERLAPS_NONRAM
;
189 /* Sanitize carve_start and carve_end to lie within our bounds */
190 carve_start
= max(carve_start
, map_start
);
191 carve_end
= min(carve_end
, map_end
);
193 /* Carving at the beginning of our map? Just move it! */
194 if (carve_start
== map_start
) {
195 if (map_end
== carve_end
) {
196 /* Full overlap, just remove map */
197 list_del(&map
->link
);
200 map
->desc
.physical_start
= carve_end
;
201 map
->desc
.virtual_start
= carve_end
;
202 map
->desc
.num_pages
= (map_end
- carve_end
)
206 return (carve_end
- carve_start
) >> EFI_PAGE_SHIFT
;
210 * Overlapping maps, just split the list map at carve_start,
211 * it will get moved or removed in the next iteration.
213 * [ map_desc |__carve_start__| newmap ]
216 /* Create a new map from [ carve_start ... map_end ] */
217 newmap
= calloc(1, sizeof(*newmap
));
218 newmap
->desc
= map
->desc
;
219 newmap
->desc
.physical_start
= carve_start
;
220 newmap
->desc
.virtual_start
= carve_start
;
221 newmap
->desc
.num_pages
= (map_end
- carve_start
) >> EFI_PAGE_SHIFT
;
222 /* Insert before current entry (descending address order) */
223 list_add_tail(&newmap
->link
, &map
->link
);
225 /* Shrink the map to [ map_start ... carve_start ] */
226 map_desc
->num_pages
= (carve_start
- map_start
) >> EFI_PAGE_SHIFT
;
228 return EFI_CARVE_LOOP_AGAIN
;
232 * efi_add_memory_map_pg() - add pages to the memory map
234 * @start: start address, must be a multiple of EFI_PAGE_SIZE
235 * @pages: number of pages to add
236 * @memory_type: type of memory added
237 * @overlap_only_ram: the memory area must overlap existing
238 * Return: status code
240 static efi_status_t
efi_add_memory_map_pg(u64 start
, u64 pages
,
242 bool overlap_only_ram
)
244 struct list_head
*lhandle
;
245 struct efi_mem_list
*newlist
;
247 uint64_t carved_pages
= 0;
248 struct efi_event
*evt
;
250 EFI_PRINT("%s: 0x%llx 0x%llx %d %s\n", __func__
,
251 start
, pages
, memory_type
, overlap_only_ram
? "yes" : "no");
253 if (memory_type
>= EFI_MAX_MEMORY_TYPE
)
254 return EFI_INVALID_PARAMETER
;
259 ++efi_memory_map_key
;
260 newlist
= calloc(1, sizeof(*newlist
));
261 newlist
->desc
.type
= memory_type
;
262 newlist
->desc
.physical_start
= start
;
263 newlist
->desc
.virtual_start
= start
;
264 newlist
->desc
.num_pages
= pages
;
266 switch (memory_type
) {
267 case EFI_RUNTIME_SERVICES_CODE
:
268 case EFI_RUNTIME_SERVICES_DATA
:
269 newlist
->desc
.attribute
= EFI_MEMORY_WB
| EFI_MEMORY_RUNTIME
;
272 newlist
->desc
.attribute
= EFI_MEMORY_RUNTIME
;
275 newlist
->desc
.attribute
= EFI_MEMORY_WB
;
279 /* Add our new map */
282 list_for_each(lhandle
, &efi_mem
) {
283 struct efi_mem_list
*lmem
;
286 lmem
= list_entry(lhandle
, struct efi_mem_list
, link
);
287 r
= efi_mem_carve_out(lmem
, &newlist
->desc
,
290 case EFI_CARVE_OVERLAPS_NONRAM
:
292 * The user requested to only have RAM overlaps,
293 * but we hit a non-RAM region. Error out.
295 return EFI_NO_MAPPING
;
296 case EFI_CARVE_NO_OVERLAP
:
297 /* Just ignore this list entry */
299 case EFI_CARVE_LOOP_AGAIN
:
301 * We split an entry, but need to loop through
302 * the list again to actually carve it.
307 /* We carved a number of pages */
314 /* The list changed, we need to start over */
318 } while (carve_again
);
320 if (overlap_only_ram
&& (carved_pages
!= pages
)) {
322 * The payload wanted to have RAM overlaps, but we overlapped
323 * with an unallocated region. Error out.
325 return EFI_NO_MAPPING
;
328 /* Add our new map */
329 list_add_tail(&newlist
->link
, &efi_mem
);
331 /* And make sure memory is listed in descending order */
334 /* Notify that the memory map was changed */
335 list_for_each_entry(evt
, &efi_events
, link
) {
338 &efi_guid_event_group_memory_map_change
)) {
339 efi_signal_event(evt
);
348 * efi_add_memory_map() - add memory area to the memory map
350 * @start: start address of the memory area
351 * @size: length in bytes of the memory area
352 * @memory_type: type of memory added
354 * Return: status code
356 * This function automatically aligns the start and size of the memory area
359 efi_status_t
efi_add_memory_map(u64 start
, u64 size
, int memory_type
)
363 pages
= efi_size_in_pages(size
+ (start
& EFI_PAGE_MASK
));
364 start
&= ~EFI_PAGE_MASK
;
366 return efi_add_memory_map_pg(start
, pages
, memory_type
, false);
370 * efi_check_allocated() - validate address to be freed
372 * Check that the address is within allocated memory:
374 * * The address must be in a range of the memory map.
375 * * The address may not point to EFI_CONVENTIONAL_MEMORY.
377 * Page alignment is not checked as this is not a requirement of
380 * @addr: address of page to be freed
381 * @must_be_allocated: return success if the page is allocated
382 * Return: status code
384 static efi_status_t
efi_check_allocated(u64 addr
, bool must_be_allocated
)
386 struct efi_mem_list
*item
;
388 list_for_each_entry(item
, &efi_mem
, link
) {
389 u64 start
= item
->desc
.physical_start
;
390 u64 end
= start
+ (item
->desc
.num_pages
<< EFI_PAGE_SHIFT
);
392 if (addr
>= start
&& addr
< end
) {
393 if (must_be_allocated
^
394 (item
->desc
.type
== EFI_CONVENTIONAL_MEMORY
))
397 return EFI_NOT_FOUND
;
401 return EFI_NOT_FOUND
;
404 static uint64_t efi_find_free_memory(uint64_t len
, uint64_t max_addr
)
406 struct list_head
*lhandle
;
409 * Prealign input max address, so we simplify our matching
410 * logic below and can just reuse it as return pointer.
412 max_addr
&= ~EFI_PAGE_MASK
;
414 list_for_each(lhandle
, &efi_mem
) {
415 struct efi_mem_list
*lmem
= list_entry(lhandle
,
416 struct efi_mem_list
, link
);
417 struct efi_mem_desc
*desc
= &lmem
->desc
;
418 uint64_t desc_len
= desc
->num_pages
<< EFI_PAGE_SHIFT
;
419 uint64_t desc_end
= desc
->physical_start
+ desc_len
;
420 uint64_t curmax
= min(max_addr
, desc_end
);
421 uint64_t ret
= curmax
- len
;
423 /* We only take memory from free RAM */
424 if (desc
->type
!= EFI_CONVENTIONAL_MEMORY
)
427 /* Out of bounds for max_addr */
428 if ((ret
+ len
) > max_addr
)
431 /* Out of bounds for upper map limit */
432 if ((ret
+ len
) > desc_end
)
435 /* Out of bounds for lower map limit */
436 if (ret
< desc
->physical_start
)
439 /* Return the highest address in this map within bounds */
447 * Allocate memory pages.
449 * @type type of allocation to be performed
450 * @memory_type usage type of the allocated memory
451 * @pages number of pages to be allocated
452 * @memory allocated memory
453 * @return status code
455 efi_status_t
efi_allocate_pages(int type
, int memory_type
,
456 efi_uintn_t pages
, uint64_t *memory
)
458 u64 len
= pages
<< EFI_PAGE_SHIFT
;
462 /* Check import parameters */
463 if (memory_type
>= EFI_PERSISTENT_MEMORY_TYPE
&&
464 memory_type
<= 0x6FFFFFFF)
465 return EFI_INVALID_PARAMETER
;
467 return EFI_INVALID_PARAMETER
;
470 case EFI_ALLOCATE_ANY_PAGES
:
472 addr
= efi_find_free_memory(len
, -1ULL);
474 return EFI_OUT_OF_RESOURCES
;
476 case EFI_ALLOCATE_MAX_ADDRESS
:
478 addr
= efi_find_free_memory(len
, *memory
);
480 return EFI_OUT_OF_RESOURCES
;
482 case EFI_ALLOCATE_ADDRESS
:
483 /* Exact address, reserve it. The addr is already in *memory. */
484 ret
= efi_check_allocated(*memory
, false);
485 if (ret
!= EFI_SUCCESS
)
486 return EFI_NOT_FOUND
;
490 /* UEFI doesn't specify other allocation types */
491 return EFI_INVALID_PARAMETER
;
494 /* Reserve that map in our memory maps */
495 ret
= efi_add_memory_map_pg(addr
, pages
, memory_type
, true);
496 if (ret
!= EFI_SUCCESS
)
497 /* Map would overlap, bail out */
498 return EFI_OUT_OF_RESOURCES
;
505 void *efi_alloc(uint64_t len
, int memory_type
)
508 uint64_t pages
= efi_size_in_pages(len
);
511 r
= efi_allocate_pages(EFI_ALLOCATE_ANY_PAGES
, memory_type
, pages
,
513 if (r
== EFI_SUCCESS
)
514 return (void*)(uintptr_t)ret
;
520 * efi_free_pages() - free memory pages
522 * @memory: start of the memory area to be freed
523 * @pages: number of pages to be freed
524 * Return: status code
526 efi_status_t
efi_free_pages(uint64_t memory
, efi_uintn_t pages
)
530 ret
= efi_check_allocated(memory
, true);
531 if (ret
!= EFI_SUCCESS
)
535 if (!memory
|| (memory
& EFI_PAGE_MASK
) || !pages
) {
536 printf("%s: illegal free 0x%llx, 0x%zx\n", __func__
,
538 return EFI_INVALID_PARAMETER
;
541 ret
= efi_add_memory_map_pg(memory
, pages
, EFI_CONVENTIONAL_MEMORY
,
543 /* Merging of adjacent free regions is missing */
545 if (ret
!= EFI_SUCCESS
)
546 return EFI_NOT_FOUND
;
552 * efi_allocate_pool - allocate memory from pool
554 * @pool_type: type of the pool from which memory is to be allocated
555 * @size: number of bytes to be allocated
556 * @buffer: allocated memory
557 * Return: status code
559 efi_status_t
efi_allocate_pool(int pool_type
, efi_uintn_t size
, void **buffer
)
563 struct efi_pool_allocation
*alloc
;
564 u64 num_pages
= efi_size_in_pages(size
+
565 sizeof(struct efi_pool_allocation
));
568 return EFI_INVALID_PARAMETER
;
575 r
= efi_allocate_pages(EFI_ALLOCATE_ANY_PAGES
, pool_type
, num_pages
,
577 if (r
== EFI_SUCCESS
) {
578 alloc
= (struct efi_pool_allocation
*)(uintptr_t)addr
;
579 alloc
->num_pages
= num_pages
;
580 alloc
->checksum
= checksum(alloc
);
581 *buffer
= alloc
->data
;
588 * efi_free_pool() - free memory from pool
590 * @buffer: start of memory to be freed
591 * Return: status code
593 efi_status_t
efi_free_pool(void *buffer
)
596 struct efi_pool_allocation
*alloc
;
599 return EFI_INVALID_PARAMETER
;
601 ret
= efi_check_allocated((uintptr_t)buffer
, true);
602 if (ret
!= EFI_SUCCESS
)
605 alloc
= container_of(buffer
, struct efi_pool_allocation
, data
);
607 /* Check that this memory was allocated by efi_allocate_pool() */
608 if (((uintptr_t)alloc
& EFI_PAGE_MASK
) ||
609 alloc
->checksum
!= checksum(alloc
)) {
610 printf("%s: illegal free 0x%p\n", __func__
, buffer
);
611 return EFI_INVALID_PARAMETER
;
613 /* Avoid double free */
616 ret
= efi_free_pages((uintptr_t)alloc
, alloc
->num_pages
);
622 * Get map describing memory usage.
624 * @memory_map_size on entry the size, in bytes, of the memory map buffer,
625 * on exit the size of the copied memory map
626 * @memory_map buffer to which the memory map is written
627 * @map_key key for the memory map
628 * @descriptor_size size of an individual memory descriptor
629 * @descriptor_version version number of the memory descriptor structure
630 * @return status code
632 efi_status_t
efi_get_memory_map(efi_uintn_t
*memory_map_size
,
633 struct efi_mem_desc
*memory_map
,
634 efi_uintn_t
*map_key
,
635 efi_uintn_t
*descriptor_size
,
636 uint32_t *descriptor_version
)
638 efi_uintn_t map_size
= 0;
640 struct list_head
*lhandle
;
641 efi_uintn_t provided_map_size
;
643 if (!memory_map_size
)
644 return EFI_INVALID_PARAMETER
;
646 provided_map_size
= *memory_map_size
;
648 list_for_each(lhandle
, &efi_mem
)
651 map_size
= map_entries
* sizeof(struct efi_mem_desc
);
653 *memory_map_size
= map_size
;
656 *descriptor_size
= sizeof(struct efi_mem_desc
);
658 if (descriptor_version
)
659 *descriptor_version
= EFI_MEMORY_DESCRIPTOR_VERSION
;
661 if (provided_map_size
< map_size
)
662 return EFI_BUFFER_TOO_SMALL
;
665 return EFI_INVALID_PARAMETER
;
667 /* Copy list into array */
668 /* Return the list in ascending order */
669 memory_map
= &memory_map
[map_entries
- 1];
670 list_for_each(lhandle
, &efi_mem
) {
671 struct efi_mem_list
*lmem
;
673 lmem
= list_entry(lhandle
, struct efi_mem_list
, link
);
674 *memory_map
= lmem
->desc
;
679 *map_key
= efi_memory_map_key
;
685 * efi_add_conventional_memory_map() - add a RAM memory area to the map
687 * @ram_start: start address of a RAM memory area
688 * @ram_end: end address of a RAM memory area
689 * @ram_top: max address to be used as conventional memory
690 * Return: status code
692 efi_status_t
efi_add_conventional_memory_map(u64 ram_start
, u64 ram_end
,
697 /* Remove partial pages */
698 ram_end
&= ~EFI_PAGE_MASK
;
699 ram_start
= (ram_start
+ EFI_PAGE_MASK
) & ~EFI_PAGE_MASK
;
701 if (ram_end
<= ram_start
) {
702 /* Invalid mapping */
703 return EFI_INVALID_PARAMETER
;
706 pages
= (ram_end
- ram_start
) >> EFI_PAGE_SHIFT
;
708 efi_add_memory_map_pg(ram_start
, pages
,
709 EFI_CONVENTIONAL_MEMORY
, false);
712 * Boards may indicate to the U-Boot memory core that they
713 * can not support memory above ram_top. Let's honor this
714 * in the efi_loader subsystem too by declaring any memory
715 * above ram_top as "already occupied by firmware".
717 if (ram_top
< ram_start
) {
718 /* ram_top is before this region, reserve all */
719 efi_add_memory_map_pg(ram_start
, pages
,
720 EFI_BOOT_SERVICES_DATA
, true);
721 } else if ((ram_top
>= ram_start
) && (ram_top
< ram_end
)) {
722 /* ram_top is inside this region, reserve parts */
723 pages
= (ram_end
- ram_top
) >> EFI_PAGE_SHIFT
;
725 efi_add_memory_map_pg(ram_top
, pages
,
726 EFI_BOOT_SERVICES_DATA
, true);
732 __weak
void efi_add_known_memory(void)
734 u64 ram_top
= board_get_usable_ram_top(0) & ~EFI_PAGE_MASK
;
738 * ram_top is just outside mapped memory. So use an offset of one for
739 * mapping the sandbox address.
741 ram_top
= (uintptr_t)map_sysmem(ram_top
- 1, 0) + 1;
743 /* Fix for 32bit targets with ram_top at 4G */
745 ram_top
= 0x100000000ULL
;
748 for (i
= 0; i
< CONFIG_NR_DRAM_BANKS
; i
++) {
749 u64 ram_end
, ram_start
;
751 ram_start
= (uintptr_t)map_sysmem(gd
->bd
->bi_dram
[i
].start
, 0);
752 ram_end
= ram_start
+ gd
->bd
->bi_dram
[i
].size
;
754 efi_add_conventional_memory_map(ram_start
, ram_end
, ram_top
);
758 /* Add memory regions for U-Boot's memory and for the runtime services code */
759 static void add_u_boot_and_runtime(void)
761 unsigned long runtime_start
, runtime_end
, runtime_pages
;
762 unsigned long runtime_mask
= EFI_PAGE_MASK
;
763 unsigned long uboot_start
, uboot_pages
;
764 unsigned long uboot_stack_size
= 16 * 1024 * 1024;
767 uboot_start
= ((uintptr_t)map_sysmem(gd
->start_addr_sp
, 0) -
768 uboot_stack_size
) & ~EFI_PAGE_MASK
;
769 uboot_pages
= ((uintptr_t)map_sysmem(gd
->ram_top
- 1, 0) -
770 uboot_start
+ EFI_PAGE_MASK
) >> EFI_PAGE_SHIFT
;
771 efi_add_memory_map_pg(uboot_start
, uboot_pages
, EFI_LOADER_DATA
,
774 #if defined(__aarch64__)
776 * Runtime Services must be 64KiB aligned according to the
777 * "AArch64 Platforms" section in the UEFI spec (2.7+).
780 runtime_mask
= SZ_64K
- 1;
784 * Add Runtime Services. We mark surrounding boottime code as runtime as
785 * well to fulfill the runtime alignment constraints but avoid padding.
787 runtime_start
= (ulong
)&__efi_runtime_start
& ~runtime_mask
;
788 runtime_end
= (ulong
)&__efi_runtime_stop
;
789 runtime_end
= (runtime_end
+ runtime_mask
) & ~runtime_mask
;
790 runtime_pages
= (runtime_end
- runtime_start
) >> EFI_PAGE_SHIFT
;
791 efi_add_memory_map_pg(runtime_start
, runtime_pages
,
792 EFI_RUNTIME_SERVICES_CODE
, false);
795 int efi_memory_init(void)
797 efi_add_known_memory();
799 add_u_boot_and_runtime();
801 #ifdef CONFIG_EFI_LOADER_BOUNCE_BUFFER
802 /* Request a 32bit 64MB bounce buffer region */
803 uint64_t efi_bounce_buffer_addr
= 0xffffffff;
805 if (efi_allocate_pages(EFI_ALLOCATE_MAX_ADDRESS
, EFI_LOADER_DATA
,
806 (64 * 1024 * 1024) >> EFI_PAGE_SHIFT
,
807 &efi_bounce_buffer_addr
) != EFI_SUCCESS
)
810 efi_bounce_buffer
= (void*)(uintptr_t)efi_bounce_buffer_addr
;