]> git.ipfire.org Git - thirdparty/linux.git/blame - kernel/memremap.c
aio: fix spectre gadget in lookup_ioctx
[thirdparty/linux.git] / kernel / memremap.c
CommitLineData
5981690d
DW
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright(c) 2015 Intel Corporation. All rights reserved. */
7d3dcf26 3#include <linux/device.h>
92281dee 4#include <linux/io.h>
0207df4f 5#include <linux/kasan.h>
41e94a85 6#include <linux/memory_hotplug.h>
bcfa4b72
MW
7#include <linux/mm.h>
8#include <linux/pfn_t.h>
5042db43
JG
9#include <linux/swap.h>
10#include <linux/swapops.h>
bcfa4b72 11#include <linux/types.h>
e7638488 12#include <linux/wait_bit.h>
bcfa4b72 13#include <linux/xarray.h>
92281dee 14
bcfa4b72 15static DEFINE_XARRAY(pgmap_array);
9476df7d
DW
16#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
17#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
18
5042db43 19#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
2b740303 20vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
5042db43
JG
21 unsigned long addr,
22 swp_entry_t entry,
23 unsigned int flags,
24 pmd_t *pmdp)
25{
26 struct page *page = device_private_entry_to_page(entry);
27
28 /*
29 * The page_fault() callback must migrate page back to system memory
30 * so that CPU can access it. This might fail for various reasons
31 * (device issue, device was unsafely unplugged, ...). When such
32 * error conditions happen, the callback must return VM_FAULT_SIGBUS.
33 *
34 * Note that because memory cgroup charges are accounted to the device
35 * memory, this should never fail because of memory restrictions (but
36 * allocation of regular system page might still fail because we are
37 * out of memory).
38 *
39 * There is a more in-depth description of what that callback can and
40 * cannot do, in include/linux/memremap.h
41 */
42 return page->pgmap->page_fault(vma, addr, page, flags, pmdp);
43}
44EXPORT_SYMBOL(device_private_entry_fault);
45#endif /* CONFIG_DEVICE_PRIVATE */
46
bcfa4b72 47static void pgmap_array_delete(struct resource *res)
ab1b597e 48{
bcfa4b72
MW
49 xa_store_range(&pgmap_array, PHYS_PFN(res->start), PHYS_PFN(res->end),
50 NULL, GFP_KERNEL);
ab1b597e 51 synchronize_rcu();
9476df7d
DW
52}
53
e7744aa2 54static unsigned long pfn_first(struct dev_pagemap *pgmap)
5c2c2587 55{
e7744aa2
LG
56 const struct resource *res = &pgmap->res;
57 struct vmem_altmap *altmap = &pgmap->altmap;
5c2c2587
DW
58 unsigned long pfn;
59
60 pfn = res->start >> PAGE_SHIFT;
e7744aa2 61 if (pgmap->altmap_valid)
5c2c2587
DW
62 pfn += vmem_altmap_offset(altmap);
63 return pfn;
64}
65
e7744aa2 66static unsigned long pfn_end(struct dev_pagemap *pgmap)
5c2c2587 67{
e7744aa2 68 const struct resource *res = &pgmap->res;
5c2c2587
DW
69
70 return (res->start + resource_size(res)) >> PAGE_SHIFT;
71}
72
949b9325
DW
73static unsigned long pfn_next(unsigned long pfn)
74{
75 if (pfn % 1024 == 0)
76 cond_resched();
77 return pfn + 1;
78}
79
5c2c2587 80#define for_each_device_pfn(pfn, map) \
949b9325 81 for (pfn = pfn_first(map); pfn < pfn_end(map); pfn = pfn_next(pfn))
5c2c2587 82
e8d51348 83static void devm_memremap_pages_release(void *data)
41e94a85 84{
e7744aa2 85 struct dev_pagemap *pgmap = data;
e8d51348 86 struct device *dev = pgmap->dev;
e7744aa2 87 struct resource *res = &pgmap->res;
9476df7d 88 resource_size_t align_start, align_size;
71389703
DW
89 unsigned long pfn;
90
e7744aa2 91 for_each_device_pfn(pfn, pgmap)
71389703 92 put_page(pfn_to_page(pfn));
9476df7d 93
5c2c2587
DW
94 if (percpu_ref_tryget_live(pgmap->ref)) {
95 dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
96 percpu_ref_put(pgmap->ref);
97 }
98
41e94a85 99 /* pages are dead and unused, undo the arch mapping */
9476df7d 100 align_start = res->start & ~(SECTION_SIZE - 1);
10a0cd6e
JS
101 align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
102 - align_start;
b5d24fda 103
f931ab47 104 mem_hotplug_begin();
e7744aa2
LG
105 arch_remove_memory(align_start, align_size, pgmap->altmap_valid ?
106 &pgmap->altmap : NULL);
0207df4f 107 kasan_remove_zero_shadow(__va(align_start), align_size);
f931ab47 108 mem_hotplug_done();
b5d24fda 109
9049771f 110 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
bcfa4b72 111 pgmap_array_delete(res);
e7744aa2
LG
112 dev_WARN_ONCE(dev, pgmap->altmap.alloc,
113 "%s: failed to free all reserved pages\n", __func__);
9476df7d
DW
114}
115
4b94ffdc
DW
116/**
117 * devm_memremap_pages - remap and provide memmap backing for the given resource
118 * @dev: hosting device for @res
e8d51348 119 * @pgmap: pointer to a struct dev_pgmap
4b94ffdc 120 *
5c2c2587 121 * Notes:
e8d51348
CH
122 * 1/ At a minimum the res, ref and type members of @pgmap must be initialized
123 * by the caller before passing it to this function
124 *
125 * 2/ The altmap field may optionally be initialized, in which case altmap_valid
126 * must be set to true
127 *
128 * 3/ pgmap.ref must be 'live' on entry and 'dead' before devm_memunmap_pages()
129 * time (or devm release event). The expected order of events is that ref has
71389703
DW
130 * been through percpu_ref_kill() before devm_memremap_pages_release(). The
131 * wait for the completion of all references being dropped and
132 * percpu_ref_exit() must occur after devm_memremap_pages_release().
5c2c2587 133 *
e8d51348 134 * 4/ res is expected to be a host memory range that could feasibly be
5c2c2587
DW
135 * treated as a "System RAM" range, i.e. not a device mmio range, but
136 * this is not enforced.
4b94ffdc 137 */
e8d51348 138void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
41e94a85 139{
ab1b597e 140 resource_size_t align_start, align_size, align_end;
e8d51348
CH
141 struct vmem_altmap *altmap = pgmap->altmap_valid ?
142 &pgmap->altmap : NULL;
949b9325 143 struct resource *res = &pgmap->res;
966cf44f 144 struct dev_pagemap *conflict_pgmap;
9049771f 145 pgprot_t pgprot = PAGE_KERNEL;
949b9325 146 int error, nid, is_ram;
5f29a77c
DW
147
148 align_start = res->start & ~(SECTION_SIZE - 1);
149 align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
150 - align_start;
15d36fec
DJ
151 align_end = align_start + align_size - 1;
152
153 conflict_pgmap = get_dev_pagemap(PHYS_PFN(align_start), NULL);
154 if (conflict_pgmap) {
155 dev_WARN(dev, "Conflicting mapping in same section\n");
156 put_dev_pagemap(conflict_pgmap);
157 return ERR_PTR(-ENOMEM);
158 }
159
160 conflict_pgmap = get_dev_pagemap(PHYS_PFN(align_end), NULL);
161 if (conflict_pgmap) {
162 dev_WARN(dev, "Conflicting mapping in same section\n");
163 put_dev_pagemap(conflict_pgmap);
164 return ERR_PTR(-ENOMEM);
165 }
166
d37a14bb
LT
167 is_ram = region_intersects(align_start, align_size,
168 IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE);
41e94a85
CH
169
170 if (is_ram == REGION_MIXED) {
171 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
172 __func__, res);
173 return ERR_PTR(-ENXIO);
174 }
175
176 if (is_ram == REGION_INTERSECTS)
177 return __va(res->start);
178
e8d51348 179 if (!pgmap->ref)
5c2c2587
DW
180 return ERR_PTR(-EINVAL);
181
4b94ffdc 182 pgmap->dev = dev;
4b94ffdc 183
bcfa4b72
MW
184 error = xa_err(xa_store_range(&pgmap_array, PHYS_PFN(res->start),
185 PHYS_PFN(res->end), pgmap, GFP_KERNEL));
9476df7d 186 if (error)
bcfa4b72 187 goto err_array;
9476df7d 188
41e94a85
CH
189 nid = dev_to_node(dev);
190 if (nid < 0)
7eff93b7 191 nid = numa_mem_id();
41e94a85 192
9049771f
DW
193 error = track_pfn_remap(NULL, &pgprot, PHYS_PFN(align_start), 0,
194 align_size);
195 if (error)
196 goto err_pfn_remap;
197
f931ab47 198 mem_hotplug_begin();
0207df4f
AR
199 error = kasan_add_zero_shadow(__va(align_start), align_size);
200 if (error) {
201 mem_hotplug_done();
202 goto err_kasan;
203 }
204
24e6d5a5 205 error = arch_add_memory(nid, align_start, align_size, altmap, false);
f1dd2cd1
MH
206 if (!error)
207 move_pfn_range_to_zone(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
208 align_start >> PAGE_SHIFT,
a99583e7 209 align_size >> PAGE_SHIFT, altmap);
f931ab47 210 mem_hotplug_done();
9476df7d
DW
211 if (error)
212 goto err_add_memory;
41e94a85 213
966cf44f
AD
214 /*
215 * Initialization of the pages has been deferred until now in order
216 * to allow us to do the work while not holding the hotplug lock.
217 */
218 memmap_init_zone_device(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
219 align_start >> PAGE_SHIFT,
220 align_size >> PAGE_SHIFT, pgmap);
221 percpu_ref_get_many(pgmap->ref, pfn_end(pgmap) - pfn_first(pgmap));
e8d51348
CH
222
223 devm_add_action(dev, devm_memremap_pages_release, pgmap);
224
41e94a85 225 return __va(res->start);
9476df7d
DW
226
227 err_add_memory:
0207df4f
AR
228 kasan_remove_zero_shadow(__va(align_start), align_size);
229 err_kasan:
9049771f
DW
230 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
231 err_pfn_remap:
bcfa4b72
MW
232 pgmap_array_delete(res);
233 err_array:
9476df7d 234 return ERR_PTR(error);
41e94a85
CH
235}
236EXPORT_SYMBOL(devm_memremap_pages);
4b94ffdc
DW
237
238unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
239{
240 /* number of pfns from base where pfn_to_page() is valid */
241 return altmap->reserve + altmap->free;
242}
243
244void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)
245{
246 altmap->alloc -= nr_pfns;
247}
248
0822acb8
CH
249/**
250 * get_dev_pagemap() - take a new live reference on the dev_pagemap for @pfn
251 * @pfn: page frame number to lookup page_map
252 * @pgmap: optional known pgmap that already has a reference
253 *
832d7aa0
CH
254 * If @pgmap is non-NULL and covers @pfn it will be returned as-is. If @pgmap
255 * is non-NULL but does not cover @pfn the reference to it will be released.
0822acb8
CH
256 */
257struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
258 struct dev_pagemap *pgmap)
259{
0822acb8
CH
260 resource_size_t phys = PFN_PHYS(pfn);
261
262 /*
832d7aa0 263 * In the cached case we're already holding a live reference.
0822acb8 264 */
832d7aa0 265 if (pgmap) {
e7744aa2 266 if (phys >= pgmap->res.start && phys <= pgmap->res.end)
832d7aa0
CH
267 return pgmap;
268 put_dev_pagemap(pgmap);
0822acb8
CH
269 }
270
271 /* fall back to slow path lookup */
272 rcu_read_lock();
bcfa4b72 273 pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
0822acb8
CH
274 if (pgmap && !percpu_ref_tryget_live(pgmap->ref))
275 pgmap = NULL;
276 rcu_read_unlock();
277
278 return pgmap;
279}
e7638488 280EXPORT_SYMBOL_GPL(get_dev_pagemap);
7b2d55d2 281
e7638488
DW
282#ifdef CONFIG_DEV_PAGEMAP_OPS
283DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
31c5bda3 284EXPORT_SYMBOL(devmap_managed_key);
e7638488
DW
285static atomic_t devmap_enable;
286
287/*
288 * Toggle the static key for ->page_free() callbacks when dev_pagemap
289 * pages go idle.
290 */
291void dev_pagemap_get_ops(void)
292{
293 if (atomic_inc_return(&devmap_enable) == 1)
294 static_branch_enable(&devmap_managed_key);
295}
296EXPORT_SYMBOL_GPL(dev_pagemap_get_ops);
297
298void dev_pagemap_put_ops(void)
299{
300 if (atomic_dec_and_test(&devmap_enable))
301 static_branch_disable(&devmap_managed_key);
302}
303EXPORT_SYMBOL_GPL(dev_pagemap_put_ops);
304
305void __put_devmap_managed_page(struct page *page)
7b2d55d2
JG
306{
307 int count = page_ref_dec_return(page);
308
309 /*
310 * If refcount is 1 then page is freed and refcount is stable as nobody
311 * holds a reference on the page.
312 */
313 if (count == 1) {
314 /* Clear Active bit in case of parallel mark_page_accessed */
315 __ClearPageActive(page);
316 __ClearPageWaiters(page);
317
c733a828 318 mem_cgroup_uncharge(page);
7b2d55d2
JG
319
320 page->pgmap->page_free(page, page->pgmap->data);
321 } else if (!count)
322 __put_page(page);
323}
31c5bda3 324EXPORT_SYMBOL(__put_devmap_managed_page);
e7638488 325#endif /* CONFIG_DEV_PAGEMAP_OPS */