]>
Commit | Line | Data |
---|---|---|
2a2c51d3 TF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (C) 2018 Intel Corporation | |
4 | * Copyright 2018 Google LLC. | |
5 | * | |
6 | * Author: Tomasz Figa <tfiga@chromium.org> | |
7 | * Author: Yong Zhi <yong.zhi@intel.com> | |
8 | */ | |
9 | ||
10 | #include <linux/vmalloc.h> | |
11 | ||
12 | #include "ipu3.h" | |
13 | #include "ipu3-css-pool.h" | |
14 | #include "ipu3-mmu.h" | |
5f5b4fa5 | 15 | #include "ipu3-dmamap.h" |
2a2c51d3 TF |
16 | |
17 | /* | |
27b795ad | 18 | * Free a buffer allocated by imgu_dmamap_alloc_buffer() |
2a2c51d3 | 19 | */ |
27b795ad | 20 | static void imgu_dmamap_free_buffer(struct page **pages, |
2a2c51d3 TF |
21 | size_t size) |
22 | { | |
23 | int count = size >> PAGE_SHIFT; | |
24 | ||
25 | while (count--) | |
26 | __free_page(pages[count]); | |
27 | kvfree(pages); | |
28 | } | |
29 | ||
30 | /* | |
31 | * Based on the implementation of __iommu_dma_alloc_pages() | |
32 | * defined in drivers/iommu/dma-iommu.c | |
33 | */ | |
17f61abb | 34 | static struct page **imgu_dmamap_alloc_buffer(size_t size, gfp_t gfp) |
2a2c51d3 TF |
35 | { |
36 | struct page **pages; | |
37 | unsigned int i = 0, count = size >> PAGE_SHIFT; | |
17f61abb | 38 | unsigned int order_mask = 1; |
2a2c51d3 TF |
39 | const gfp_t high_order_gfp = __GFP_NOWARN | __GFP_NORETRY; |
40 | ||
41 | /* Allocate mem for array of page ptrs */ | |
42 | pages = kvmalloc_array(count, sizeof(*pages), GFP_KERNEL); | |
43 | ||
44 | if (!pages) | |
45 | return NULL; | |
46 | ||
2a2c51d3 TF |
47 | gfp |= __GFP_HIGHMEM | __GFP_ZERO; |
48 | ||
49 | while (count) { | |
50 | struct page *page = NULL; | |
51 | unsigned int order_size; | |
52 | ||
53 | for (order_mask &= (2U << __fls(count)) - 1; | |
54 | order_mask; order_mask &= ~order_size) { | |
55 | unsigned int order = __fls(order_mask); | |
56 | ||
57 | order_size = 1U << order; | |
58 | page = alloc_pages((order_mask - order_size) ? | |
59 | gfp | high_order_gfp : gfp, order); | |
60 | if (!page) | |
61 | continue; | |
62 | if (!order) | |
63 | break; | |
64 | if (!PageCompound(page)) { | |
65 | split_page(page, order); | |
66 | break; | |
67 | } | |
68 | ||
69 | __free_pages(page, order); | |
70 | } | |
71 | if (!page) { | |
27b795ad | 72 | imgu_dmamap_free_buffer(pages, i << PAGE_SHIFT); |
2a2c51d3 TF |
73 | return NULL; |
74 | } | |
75 | count -= order_size; | |
76 | while (order_size--) | |
77 | pages[i++] = page++; | |
78 | } | |
79 | ||
80 | return pages; | |
81 | } | |
82 | ||
83 | /** | |
27b795ad | 84 | * imgu_dmamap_alloc - allocate and map a buffer into KVA |
2a2c51d3 TF |
85 | * @imgu: struct device pointer |
86 | * @map: struct to store mapping variables | |
87 | * @len: size required | |
88 | * | |
89 | * Returns: | |
90 | * KVA on success | |
91 | * %NULL on failure | |
92 | */ | |
27b795ad | 93 | void *imgu_dmamap_alloc(struct imgu_device *imgu, struct imgu_css_map *map, |
2a2c51d3 TF |
94 | size_t len) |
95 | { | |
96 | unsigned long shift = iova_shift(&imgu->iova_domain); | |
2a2c51d3 TF |
97 | struct device *dev = &imgu->pci_dev->dev; |
98 | size_t size = PAGE_ALIGN(len); | |
f8092aa1 | 99 | int count = size >> PAGE_SHIFT; |
2a2c51d3 TF |
100 | struct page **pages; |
101 | dma_addr_t iovaddr; | |
102 | struct iova *iova; | |
103 | int i, rval; | |
104 | ||
105 | dev_dbg(dev, "%s: allocating %zu\n", __func__, size); | |
106 | ||
107 | iova = alloc_iova(&imgu->iova_domain, size >> shift, | |
108 | imgu->mmu->aperture_end >> shift, 0); | |
109 | if (!iova) | |
110 | return NULL; | |
111 | ||
17f61abb | 112 | pages = imgu_dmamap_alloc_buffer(size, GFP_KERNEL); |
2a2c51d3 TF |
113 | if (!pages) |
114 | goto out_free_iova; | |
115 | ||
116 | /* Call IOMMU driver to setup pgt */ | |
117 | iovaddr = iova_dma_addr(&imgu->iova_domain, iova); | |
f8092aa1 | 118 | for (i = 0; i < count; ++i) { |
27b795ad | 119 | rval = imgu_mmu_map(imgu->mmu, iovaddr, |
2a2c51d3 TF |
120 | page_to_phys(pages[i]), PAGE_SIZE); |
121 | if (rval) | |
122 | goto out_unmap; | |
123 | ||
124 | iovaddr += PAGE_SIZE; | |
125 | } | |
126 | ||
f8092aa1 CH |
127 | map->vaddr = vmap(pages, count, VM_USERMAP, PAGE_KERNEL); |
128 | if (!map->vaddr) | |
2a2c51d3 TF |
129 | goto out_unmap; |
130 | ||
f8092aa1 | 131 | map->pages = pages; |
2a2c51d3 TF |
132 | map->size = size; |
133 | map->daddr = iova_dma_addr(&imgu->iova_domain, iova); | |
2a2c51d3 TF |
134 | |
135 | dev_dbg(dev, "%s: allocated %zu @ IOVA %pad @ VA %p\n", __func__, | |
f8092aa1 | 136 | size, &map->daddr, map->vaddr); |
2a2c51d3 | 137 | |
f8092aa1 | 138 | return map->vaddr; |
2a2c51d3 TF |
139 | |
140 | out_unmap: | |
27b795ad YZ |
141 | imgu_dmamap_free_buffer(pages, size); |
142 | imgu_mmu_unmap(imgu->mmu, iova_dma_addr(&imgu->iova_domain, iova), | |
2a2c51d3 | 143 | i * PAGE_SIZE); |
2a2c51d3 TF |
144 | |
145 | out_free_iova: | |
146 | __free_iova(&imgu->iova_domain, iova); | |
147 | ||
148 | return NULL; | |
149 | } | |
150 | ||
27b795ad | 151 | void imgu_dmamap_unmap(struct imgu_device *imgu, struct imgu_css_map *map) |
2a2c51d3 TF |
152 | { |
153 | struct iova *iova; | |
154 | ||
155 | iova = find_iova(&imgu->iova_domain, | |
156 | iova_pfn(&imgu->iova_domain, map->daddr)); | |
157 | if (WARN_ON(!iova)) | |
158 | return; | |
159 | ||
27b795ad | 160 | imgu_mmu_unmap(imgu->mmu, iova_dma_addr(&imgu->iova_domain, iova), |
2a2c51d3 TF |
161 | iova_size(iova) << iova_shift(&imgu->iova_domain)); |
162 | ||
163 | __free_iova(&imgu->iova_domain, iova); | |
164 | } | |
165 | ||
166 | /* | |
27b795ad | 167 | * Counterpart of imgu_dmamap_alloc |
2a2c51d3 | 168 | */ |
27b795ad | 169 | void imgu_dmamap_free(struct imgu_device *imgu, struct imgu_css_map *map) |
2a2c51d3 | 170 | { |
2a2c51d3 TF |
171 | dev_dbg(&imgu->pci_dev->dev, "%s: freeing %zu @ IOVA %pad @ VA %p\n", |
172 | __func__, map->size, &map->daddr, map->vaddr); | |
173 | ||
174 | if (!map->vaddr) | |
175 | return; | |
176 | ||
27b795ad | 177 | imgu_dmamap_unmap(imgu, map); |
2a2c51d3 | 178 | |
2a2c51d3 | 179 | vunmap(map->vaddr); |
f8092aa1 | 180 | imgu_dmamap_free_buffer(map->pages, map->size); |
2a2c51d3 TF |
181 | map->vaddr = NULL; |
182 | } | |
183 | ||
27b795ad YZ |
184 | int imgu_dmamap_map_sg(struct imgu_device *imgu, struct scatterlist *sglist, |
185 | int nents, struct imgu_css_map *map) | |
2a2c51d3 TF |
186 | { |
187 | unsigned long shift = iova_shift(&imgu->iova_domain); | |
188 | struct scatterlist *sg; | |
189 | struct iova *iova; | |
190 | size_t size = 0; | |
191 | int i; | |
192 | ||
193 | for_each_sg(sglist, sg, nents, i) { | |
194 | if (sg->offset) | |
195 | return -EINVAL; | |
196 | ||
197 | if (i != nents - 1 && !PAGE_ALIGNED(sg->length)) | |
198 | return -EINVAL; | |
199 | ||
200 | size += sg->length; | |
201 | } | |
202 | ||
203 | size = iova_align(&imgu->iova_domain, size); | |
204 | dev_dbg(&imgu->pci_dev->dev, "dmamap: mapping sg %d entries, %zu pages\n", | |
205 | nents, size >> shift); | |
206 | ||
207 | iova = alloc_iova(&imgu->iova_domain, size >> shift, | |
208 | imgu->mmu->aperture_end >> shift, 0); | |
209 | if (!iova) | |
210 | return -ENOMEM; | |
211 | ||
212 | dev_dbg(&imgu->pci_dev->dev, "dmamap: iova low pfn %lu, high pfn %lu\n", | |
213 | iova->pfn_lo, iova->pfn_hi); | |
214 | ||
27b795ad | 215 | if (imgu_mmu_map_sg(imgu->mmu, iova_dma_addr(&imgu->iova_domain, iova), |
2a2c51d3 TF |
216 | sglist, nents) < size) |
217 | goto out_fail; | |
218 | ||
219 | memset(map, 0, sizeof(*map)); | |
220 | map->daddr = iova_dma_addr(&imgu->iova_domain, iova); | |
221 | map->size = size; | |
222 | ||
223 | return 0; | |
224 | ||
225 | out_fail: | |
226 | __free_iova(&imgu->iova_domain, iova); | |
227 | ||
228 | return -EFAULT; | |
229 | } | |
230 | ||
27b795ad | 231 | int imgu_dmamap_init(struct imgu_device *imgu) |
2a2c51d3 TF |
232 | { |
233 | unsigned long order, base_pfn; | |
234 | int ret = iova_cache_get(); | |
235 | ||
236 | if (ret) | |
237 | return ret; | |
238 | ||
17f61abb | 239 | order = __ffs(IPU3_PAGE_SIZE); |
2a2c51d3 TF |
240 | base_pfn = max_t(unsigned long, 1, imgu->mmu->aperture_start >> order); |
241 | init_iova_domain(&imgu->iova_domain, 1UL << order, base_pfn); | |
242 | ||
243 | return 0; | |
244 | } | |
245 | ||
27b795ad | 246 | void imgu_dmamap_exit(struct imgu_device *imgu) |
2a2c51d3 TF |
247 | { |
248 | put_iova_domain(&imgu->iova_domain); | |
249 | iova_cache_put(); | |
250 | } |