1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2012 ARM Ltd.
4 * Copyright (c) 2014 The Linux Foundation
6 #include <linux/dma-direct.h>
7 #include <linux/dma-noncoherent.h>
8 #include <linux/dma-contiguous.h>
9 #include <linux/init.h>
10 #include <linux/genalloc.h>
11 #include <linux/slab.h>
12 #include <linux/vmalloc.h>
14 struct page
**dma_common_find_pages(void *cpu_addr
)
16 struct vm_struct
*area
= find_vm_area(cpu_addr
);
18 if (!area
|| area
->flags
!= VM_DMA_COHERENT
)
24 * Remaps an array of PAGE_SIZE pages into another vm_area.
25 * Cannot be used in non-sleeping contexts
27 void *dma_common_pages_remap(struct page
**pages
, size_t size
,
28 pgprot_t prot
, const void *caller
)
32 vaddr
= vmap(pages
, size
>> PAGE_SHIFT
, VM_DMA_COHERENT
, prot
);
34 find_vm_area(vaddr
)->pages
= pages
;
39 * Remaps an allocated contiguous region into another vm_area.
40 * Cannot be used in non-sleeping contexts
42 void *dma_common_contiguous_remap(struct page
*page
, size_t size
,
43 pgprot_t prot
, const void *caller
)
45 int count
= size
>> PAGE_SHIFT
;
50 pages
= kmalloc_array(count
, sizeof(struct page
*), GFP_KERNEL
);
53 for (i
= 0; i
< count
; i
++)
54 pages
[i
] = nth_page(page
, i
);
55 vaddr
= vmap(pages
, count
, VM_DMA_COHERENT
, prot
);
62 * Unmaps a range previously mapped by dma_common_*_remap
64 void dma_common_free_remap(void *cpu_addr
, size_t size
)
66 struct vm_struct
*area
= find_vm_area(cpu_addr
);
68 if (!area
|| area
->flags
!= VM_DMA_COHERENT
) {
69 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr
);
73 unmap_kernel_range((unsigned long)cpu_addr
, PAGE_ALIGN(size
));
77 #ifdef CONFIG_DMA_DIRECT_REMAP
78 static struct gen_pool
*atomic_pool __ro_after_init
;
80 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
81 static size_t atomic_pool_size __initdata
= DEFAULT_DMA_COHERENT_POOL_SIZE
;
83 static int __init
early_coherent_pool(char *p
)
85 atomic_pool_size
= memparse(p
, &p
);
88 early_param("coherent_pool", early_coherent_pool
);
90 static gfp_t
dma_atomic_pool_gfp(void)
92 if (IS_ENABLED(CONFIG_ZONE_DMA
))
94 if (IS_ENABLED(CONFIG_ZONE_DMA32
))
99 static int __init
dma_atomic_pool_init(void)
101 unsigned int pool_size_order
= get_order(atomic_pool_size
);
102 unsigned long nr_pages
= atomic_pool_size
>> PAGE_SHIFT
;
107 if (dev_get_cma_area(NULL
))
108 page
= dma_alloc_from_contiguous(NULL
, nr_pages
,
109 pool_size_order
, false);
111 page
= alloc_pages(dma_atomic_pool_gfp(), pool_size_order
);
115 arch_dma_prep_coherent(page
, atomic_pool_size
);
117 atomic_pool
= gen_pool_create(PAGE_SHIFT
, -1);
121 addr
= dma_common_contiguous_remap(page
, atomic_pool_size
,
122 pgprot_dmacoherent(PAGE_KERNEL
),
123 __builtin_return_address(0));
125 goto destroy_genpool
;
127 ret
= gen_pool_add_virt(atomic_pool
, (unsigned long)addr
,
128 page_to_phys(page
), atomic_pool_size
, -1);
131 gen_pool_set_algo(atomic_pool
, gen_pool_first_fit_order_align
, NULL
);
133 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
134 atomic_pool_size
/ 1024);
138 dma_common_free_remap(addr
, atomic_pool_size
);
140 gen_pool_destroy(atomic_pool
);
143 if (!dma_release_from_contiguous(NULL
, page
, nr_pages
))
144 __free_pages(page
, pool_size_order
);
146 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
147 atomic_pool_size
/ 1024);
150 postcore_initcall(dma_atomic_pool_init
);
152 bool dma_in_atomic_pool(void *start
, size_t size
)
154 if (unlikely(!atomic_pool
))
157 return gen_pool_has_addr(atomic_pool
, (unsigned long)start
, size
);
160 void *dma_alloc_from_pool(size_t size
, struct page
**ret_page
, gfp_t flags
)
166 WARN(1, "coherent pool not initialised!\n");
170 val
= gen_pool_alloc(atomic_pool
, size
);
172 phys_addr_t phys
= gen_pool_virt_to_phys(atomic_pool
, val
);
174 *ret_page
= pfn_to_page(__phys_to_pfn(phys
));
176 memset(ptr
, 0, size
);
182 bool dma_free_from_pool(void *start
, size_t size
)
184 if (!dma_in_atomic_pool(start
, size
))
186 gen_pool_free(atomic_pool
, (unsigned long)start
, size
);
189 #endif /* CONFIG_DMA_DIRECT_REMAP */