]>
Commit | Line | Data |
---|---|---|
5b497af4 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e1455744 | 2 | /* |
cd03412a | 3 | * Copyright(c) 2013-2016 Intel Corporation. All rights reserved. |
e1455744 | 4 | */ |
ac515c08 | 5 | #include <linux/memremap.h> |
e1455744 DW |
6 | #include <linux/blkdev.h> |
7 | #include <linux/device.h> | |
8 | #include <linux/genhd.h> | |
9 | #include <linux/sizes.h> | |
10 | #include <linux/slab.h> | |
11 | #include <linux/fs.h> | |
12 | #include <linux/mm.h> | |
13 | #include "nd-core.h" | |
14 | #include "pfn.h" | |
15 | #include "nd.h" | |
16 | ||
17 | static void nd_pfn_release(struct device *dev) | |
18 | { | |
19 | struct nd_region *nd_region = to_nd_region(dev->parent); | |
20 | struct nd_pfn *nd_pfn = to_nd_pfn(dev); | |
21 | ||
426824d6 | 22 | dev_dbg(dev, "trace\n"); |
e1455744 DW |
23 | nd_detach_ndns(&nd_pfn->dev, &nd_pfn->ndns); |
24 | ida_simple_remove(&nd_region->pfn_ida, nd_pfn->id); | |
25 | kfree(nd_pfn->uuid); | |
26 | kfree(nd_pfn); | |
27 | } | |
28 | ||
29 | static struct device_type nd_pfn_device_type = { | |
30 | .name = "nd_pfn", | |
31 | .release = nd_pfn_release, | |
32 | }; | |
33 | ||
34 | bool is_nd_pfn(struct device *dev) | |
35 | { | |
36 | return dev ? dev->type == &nd_pfn_device_type : false; | |
37 | } | |
38 | EXPORT_SYMBOL(is_nd_pfn); | |
39 | ||
40 | struct nd_pfn *to_nd_pfn(struct device *dev) | |
41 | { | |
42 | struct nd_pfn *nd_pfn = container_of(dev, struct nd_pfn, dev); | |
43 | ||
44 | WARN_ON(!is_nd_pfn(dev)); | |
45 | return nd_pfn; | |
46 | } | |
47 | EXPORT_SYMBOL(to_nd_pfn); | |
48 | ||
49 | static ssize_t mode_show(struct device *dev, | |
50 | struct device_attribute *attr, char *buf) | |
51 | { | |
cd03412a | 52 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
53 | |
54 | switch (nd_pfn->mode) { | |
55 | case PFN_MODE_RAM: | |
56 | return sprintf(buf, "ram\n"); | |
57 | case PFN_MODE_PMEM: | |
58 | return sprintf(buf, "pmem\n"); | |
59 | default: | |
60 | return sprintf(buf, "none\n"); | |
61 | } | |
62 | } | |
63 | ||
64 | static ssize_t mode_store(struct device *dev, | |
65 | struct device_attribute *attr, const char *buf, size_t len) | |
66 | { | |
cd03412a | 67 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
68 | ssize_t rc = 0; |
69 | ||
70 | device_lock(dev); | |
71 | nvdimm_bus_lock(dev); | |
72 | if (dev->driver) | |
73 | rc = -EBUSY; | |
74 | else { | |
75 | size_t n = len - 1; | |
76 | ||
77 | if (strncmp(buf, "pmem\n", n) == 0 | |
78 | || strncmp(buf, "pmem", n) == 0) { | |
d2c0f041 | 79 | nd_pfn->mode = PFN_MODE_PMEM; |
e1455744 DW |
80 | } else if (strncmp(buf, "ram\n", n) == 0 |
81 | || strncmp(buf, "ram", n) == 0) | |
82 | nd_pfn->mode = PFN_MODE_RAM; | |
83 | else if (strncmp(buf, "none\n", n) == 0 | |
84 | || strncmp(buf, "none", n) == 0) | |
85 | nd_pfn->mode = PFN_MODE_NONE; | |
86 | else | |
87 | rc = -EINVAL; | |
88 | } | |
426824d6 DW |
89 | dev_dbg(dev, "result: %zd wrote: %s%s", rc, buf, |
90 | buf[len - 1] == '\n' ? "" : "\n"); | |
e1455744 DW |
91 | nvdimm_bus_unlock(dev); |
92 | device_unlock(dev); | |
93 | ||
94 | return rc ? rc : len; | |
95 | } | |
96 | static DEVICE_ATTR_RW(mode); | |
97 | ||
315c5625 DW |
98 | static ssize_t align_show(struct device *dev, |
99 | struct device_attribute *attr, char *buf) | |
100 | { | |
cd03412a | 101 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
315c5625 | 102 | |
af7d9f0c | 103 | return sprintf(buf, "%ld\n", nd_pfn->align); |
315c5625 DW |
104 | } |
105 | ||
1fdadbeb OH |
106 | static const unsigned long *nd_pfn_supported_alignments(void) |
107 | { | |
108 | /* | |
109 | * This needs to be a non-static variable because the *_SIZE | |
110 | * macros aren't always constants. | |
111 | */ | |
112 | const unsigned long supported_alignments[] = { | |
113 | PAGE_SIZE, | |
114 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
115 | HPAGE_PMD_SIZE, | |
116 | #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD | |
117 | HPAGE_PUD_SIZE, | |
118 | #endif | |
119 | #endif | |
120 | 0, | |
121 | }; | |
122 | static unsigned long data[ARRAY_SIZE(supported_alignments)]; | |
123 | ||
124 | memcpy(data, supported_alignments, sizeof(data)); | |
125 | ||
126 | return data; | |
127 | } | |
128 | ||
315c5625 DW |
129 | static ssize_t align_store(struct device *dev, |
130 | struct device_attribute *attr, const char *buf, size_t len) | |
131 | { | |
cd03412a | 132 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
315c5625 DW |
133 | ssize_t rc; |
134 | ||
135 | device_lock(dev); | |
136 | nvdimm_bus_lock(dev); | |
f13d2b61 DW |
137 | rc = nd_size_select_store(dev, buf, &nd_pfn->align, |
138 | nd_pfn_supported_alignments()); | |
426824d6 DW |
139 | dev_dbg(dev, "result: %zd wrote: %s%s", rc, buf, |
140 | buf[len - 1] == '\n' ? "" : "\n"); | |
315c5625 DW |
141 | nvdimm_bus_unlock(dev); |
142 | device_unlock(dev); | |
143 | ||
144 | return rc ? rc : len; | |
145 | } | |
146 | static DEVICE_ATTR_RW(align); | |
147 | ||
e1455744 DW |
148 | static ssize_t uuid_show(struct device *dev, |
149 | struct device_attribute *attr, char *buf) | |
150 | { | |
cd03412a | 151 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
152 | |
153 | if (nd_pfn->uuid) | |
154 | return sprintf(buf, "%pUb\n", nd_pfn->uuid); | |
155 | return sprintf(buf, "\n"); | |
156 | } | |
157 | ||
158 | static ssize_t uuid_store(struct device *dev, | |
159 | struct device_attribute *attr, const char *buf, size_t len) | |
160 | { | |
cd03412a | 161 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
162 | ssize_t rc; |
163 | ||
164 | device_lock(dev); | |
165 | rc = nd_uuid_store(dev, &nd_pfn->uuid, buf, len); | |
426824d6 DW |
166 | dev_dbg(dev, "result: %zd wrote: %s%s", rc, buf, |
167 | buf[len - 1] == '\n' ? "" : "\n"); | |
e1455744 DW |
168 | device_unlock(dev); |
169 | ||
170 | return rc ? rc : len; | |
171 | } | |
172 | static DEVICE_ATTR_RW(uuid); | |
173 | ||
174 | static ssize_t namespace_show(struct device *dev, | |
175 | struct device_attribute *attr, char *buf) | |
176 | { | |
cd03412a | 177 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
178 | ssize_t rc; |
179 | ||
180 | nvdimm_bus_lock(dev); | |
181 | rc = sprintf(buf, "%s\n", nd_pfn->ndns | |
182 | ? dev_name(&nd_pfn->ndns->dev) : ""); | |
183 | nvdimm_bus_unlock(dev); | |
184 | return rc; | |
185 | } | |
186 | ||
187 | static ssize_t namespace_store(struct device *dev, | |
188 | struct device_attribute *attr, const char *buf, size_t len) | |
189 | { | |
cd03412a | 190 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
e1455744 DW |
191 | ssize_t rc; |
192 | ||
e1455744 | 193 | device_lock(dev); |
4ca8b57a | 194 | nvdimm_bus_lock(dev); |
e1455744 | 195 | rc = nd_namespace_store(dev, &nd_pfn->ndns, buf, len); |
426824d6 DW |
196 | dev_dbg(dev, "result: %zd wrote: %s%s", rc, buf, |
197 | buf[len - 1] == '\n' ? "" : "\n"); | |
e1455744 | 198 | nvdimm_bus_unlock(dev); |
4ca8b57a | 199 | device_unlock(dev); |
e1455744 DW |
200 | |
201 | return rc; | |
202 | } | |
203 | static DEVICE_ATTR_RW(namespace); | |
204 | ||
f6ed58c7 DW |
205 | static ssize_t resource_show(struct device *dev, |
206 | struct device_attribute *attr, char *buf) | |
207 | { | |
cd03412a | 208 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
f6ed58c7 DW |
209 | ssize_t rc; |
210 | ||
211 | device_lock(dev); | |
212 | if (dev->driver) { | |
213 | struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; | |
214 | u64 offset = __le64_to_cpu(pfn_sb->dataoff); | |
215 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
216 | u32 start_pad = __le32_to_cpu(pfn_sb->start_pad); | |
217 | struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); | |
218 | ||
219 | rc = sprintf(buf, "%#llx\n", (unsigned long long) nsio->res.start | |
220 | + start_pad + offset); | |
221 | } else { | |
222 | /* no address to convey if the pfn instance is disabled */ | |
223 | rc = -ENXIO; | |
224 | } | |
225 | device_unlock(dev); | |
226 | ||
227 | return rc; | |
228 | } | |
229 | static DEVICE_ATTR_RO(resource); | |
230 | ||
231 | static ssize_t size_show(struct device *dev, | |
232 | struct device_attribute *attr, char *buf) | |
233 | { | |
cd03412a | 234 | struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); |
f6ed58c7 DW |
235 | ssize_t rc; |
236 | ||
237 | device_lock(dev); | |
238 | if (dev->driver) { | |
239 | struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; | |
240 | u64 offset = __le64_to_cpu(pfn_sb->dataoff); | |
241 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
242 | u32 start_pad = __le32_to_cpu(pfn_sb->start_pad); | |
243 | u32 end_trunc = __le32_to_cpu(pfn_sb->end_trunc); | |
244 | struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); | |
245 | ||
246 | rc = sprintf(buf, "%llu\n", (unsigned long long) | |
247 | resource_size(&nsio->res) - start_pad | |
248 | - end_trunc - offset); | |
249 | } else { | |
250 | /* no size to convey if the pfn instance is disabled */ | |
251 | rc = -ENXIO; | |
252 | } | |
253 | device_unlock(dev); | |
254 | ||
255 | return rc; | |
256 | } | |
257 | static DEVICE_ATTR_RO(size); | |
258 | ||
1fdadbeb OH |
259 | static ssize_t supported_alignments_show(struct device *dev, |
260 | struct device_attribute *attr, char *buf) | |
261 | { | |
262 | return nd_size_select_show(0, nd_pfn_supported_alignments(), buf); | |
263 | } | |
264 | static DEVICE_ATTR_RO(supported_alignments); | |
265 | ||
e1455744 DW |
266 | static struct attribute *nd_pfn_attributes[] = { |
267 | &dev_attr_mode.attr, | |
268 | &dev_attr_namespace.attr, | |
269 | &dev_attr_uuid.attr, | |
315c5625 | 270 | &dev_attr_align.attr, |
f6ed58c7 DW |
271 | &dev_attr_resource.attr, |
272 | &dev_attr_size.attr, | |
1fdadbeb | 273 | &dev_attr_supported_alignments.attr, |
e1455744 DW |
274 | NULL, |
275 | }; | |
276 | ||
26417ae4 DW |
277 | static umode_t pfn_visible(struct kobject *kobj, struct attribute *a, int n) |
278 | { | |
279 | if (a == &dev_attr_resource.attr) | |
280 | return 0400; | |
281 | return a->mode; | |
282 | } | |
283 | ||
cd03412a | 284 | struct attribute_group nd_pfn_attribute_group = { |
e1455744 | 285 | .attrs = nd_pfn_attributes, |
26417ae4 | 286 | .is_visible = pfn_visible, |
e1455744 DW |
287 | }; |
288 | ||
289 | static const struct attribute_group *nd_pfn_attribute_groups[] = { | |
290 | &nd_pfn_attribute_group, | |
291 | &nd_device_attribute_group, | |
292 | &nd_numa_attribute_group, | |
293 | NULL, | |
294 | }; | |
295 | ||
cd03412a | 296 | struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn, |
e1455744 DW |
297 | struct nd_namespace_common *ndns) |
298 | { | |
0cbfeef2 | 299 | struct device *dev; |
e1455744 | 300 | |
cd03412a | 301 | if (!nd_pfn) |
e1455744 DW |
302 | return NULL; |
303 | ||
cd03412a | 304 | nd_pfn->mode = PFN_MODE_NONE; |
0dd69643 | 305 | nd_pfn->align = PFN_DEFAULT_ALIGNMENT; |
cd03412a DW |
306 | dev = &nd_pfn->dev; |
307 | device_initialize(&nd_pfn->dev); | |
308 | if (ndns && !__nd_attach_ndns(&nd_pfn->dev, ndns, &nd_pfn->ndns)) { | |
426824d6 DW |
309 | dev_dbg(&ndns->dev, "failed, already claimed by %s\n", |
310 | dev_name(ndns->claim)); | |
cd03412a | 311 | put_device(dev); |
e1455744 | 312 | return NULL; |
cd03412a DW |
313 | } |
314 | return dev; | |
315 | } | |
316 | ||
317 | static struct nd_pfn *nd_pfn_alloc(struct nd_region *nd_region) | |
318 | { | |
319 | struct nd_pfn *nd_pfn; | |
320 | struct device *dev; | |
e1455744 DW |
321 | |
322 | nd_pfn = kzalloc(sizeof(*nd_pfn), GFP_KERNEL); | |
323 | if (!nd_pfn) | |
324 | return NULL; | |
325 | ||
326 | nd_pfn->id = ida_simple_get(&nd_region->pfn_ida, 0, 0, GFP_KERNEL); | |
327 | if (nd_pfn->id < 0) { | |
328 | kfree(nd_pfn); | |
329 | return NULL; | |
330 | } | |
331 | ||
e1455744 DW |
332 | dev = &nd_pfn->dev; |
333 | dev_set_name(dev, "pfn%d.%d", nd_region->id, nd_pfn->id); | |
e1455744 | 334 | dev->groups = nd_pfn_attribute_groups; |
cd03412a DW |
335 | dev->type = &nd_pfn_device_type; |
336 | dev->parent = &nd_region->dev; | |
337 | ||
338 | return nd_pfn; | |
e1455744 DW |
339 | } |
340 | ||
341 | struct device *nd_pfn_create(struct nd_region *nd_region) | |
342 | { | |
cd03412a DW |
343 | struct nd_pfn *nd_pfn; |
344 | struct device *dev; | |
345 | ||
c9e582aa | 346 | if (!is_memory(&nd_region->dev)) |
cd03412a | 347 | return NULL; |
e1455744 | 348 | |
cd03412a DW |
349 | nd_pfn = nd_pfn_alloc(nd_region); |
350 | dev = nd_pfn_devinit(nd_pfn, NULL); | |
e1455744 | 351 | |
cd03412a | 352 | __nd_device_register(dev); |
e1455744 DW |
353 | return dev; |
354 | } | |
355 | ||
48af2f7e VV |
356 | /* |
357 | * nd_pfn_clear_memmap_errors() clears any errors in the volatile memmap | |
358 | * space associated with the namespace. If the memmap is set to DRAM, then | |
359 | * this is a no-op. Since the memmap area is freshly initialized during | |
360 | * probe, we have an opportunity to clear any badblocks in this area. | |
361 | */ | |
362 | static int nd_pfn_clear_memmap_errors(struct nd_pfn *nd_pfn) | |
363 | { | |
364 | struct nd_region *nd_region = to_nd_region(nd_pfn->dev.parent); | |
365 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
366 | void *zero_page = page_address(ZERO_PAGE(0)); | |
367 | struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; | |
368 | int num_bad, meta_num, rc, bb_present; | |
369 | sector_t first_bad, meta_start; | |
370 | struct nd_namespace_io *nsio; | |
371 | ||
372 | if (nd_pfn->mode != PFN_MODE_PMEM) | |
373 | return 0; | |
374 | ||
375 | nsio = to_nd_namespace_io(&ndns->dev); | |
376 | meta_start = (SZ_4K + sizeof(*pfn_sb)) >> 9; | |
377 | meta_num = (le64_to_cpu(pfn_sb->dataoff) >> 9) - meta_start; | |
378 | ||
379 | do { | |
380 | unsigned long zero_len; | |
381 | u64 nsoff; | |
382 | ||
383 | bb_present = badblocks_check(&nd_region->bb, meta_start, | |
384 | meta_num, &first_bad, &num_bad); | |
385 | if (bb_present) { | |
72deb455 | 386 | dev_dbg(&nd_pfn->dev, "meta: %x badblocks at %llx\n", |
48af2f7e VV |
387 | num_bad, first_bad); |
388 | nsoff = ALIGN_DOWN((nd_region->ndr_start | |
389 | + (first_bad << 9)) - nsio->res.start, | |
390 | PAGE_SIZE); | |
391 | zero_len = ALIGN(num_bad << 9, PAGE_SIZE); | |
392 | while (zero_len) { | |
393 | unsigned long chunk = min(zero_len, PAGE_SIZE); | |
394 | ||
395 | rc = nvdimm_write_bytes(ndns, nsoff, zero_page, | |
396 | chunk, 0); | |
397 | if (rc) | |
398 | break; | |
399 | ||
400 | zero_len -= chunk; | |
401 | nsoff += chunk; | |
402 | } | |
403 | if (rc) { | |
404 | dev_err(&nd_pfn->dev, | |
72deb455 | 405 | "error clearing %x badblocks at %llx\n", |
48af2f7e VV |
406 | num_bad, first_bad); |
407 | return rc; | |
408 | } | |
409 | } | |
410 | } while (bb_present); | |
411 | ||
412 | return 0; | |
413 | } | |
414 | ||
7e3e888d DW |
415 | /** |
416 | * nd_pfn_validate - read and validate info-block | |
417 | * @nd_pfn: fsdax namespace runtime state / properties | |
418 | * @sig: 'devdax' or 'fsdax' signature | |
419 | * | |
420 | * Upon return the info-block buffer contents (->pfn_sb) are | |
421 | * indeterminate when validation fails, and a coherent info-block | |
422 | * otherwise. | |
423 | */ | |
c5ed9268 | 424 | int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) |
e1455744 | 425 | { |
e1455744 | 426 | u64 checksum, offset; |
1ee6667c | 427 | enum nd_pfn_mode mode; |
a34d5e8a | 428 | struct nd_namespace_io *nsio; |
19deaa21 | 429 | unsigned long align, start_pad; |
a34d5e8a DW |
430 | struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; |
431 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
432 | const u8 *parent_uuid = nd_dev_to_uuid(&ndns->dev); | |
e1455744 DW |
433 | |
434 | if (!pfn_sb || !ndns) | |
435 | return -ENODEV; | |
436 | ||
c9e582aa | 437 | if (!is_memory(nd_pfn->dev.parent)) |
e1455744 DW |
438 | return -ENODEV; |
439 | ||
3ae3d67b | 440 | if (nvdimm_read_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb), 0)) |
e1455744 DW |
441 | return -ENXIO; |
442 | ||
c5ed9268 | 443 | if (memcmp(pfn_sb->signature, sig, PFN_SIG_LEN) != 0) |
e1455744 DW |
444 | return -ENODEV; |
445 | ||
446 | checksum = le64_to_cpu(pfn_sb->checksum); | |
447 | pfn_sb->checksum = 0; | |
448 | if (checksum != nd_sb_checksum((struct nd_gen_sb *) pfn_sb)) | |
449 | return -ENODEV; | |
450 | pfn_sb->checksum = cpu_to_le64(checksum); | |
451 | ||
a34d5e8a DW |
452 | if (memcmp(pfn_sb->parent_uuid, parent_uuid, 16) != 0) |
453 | return -ENODEV; | |
454 | ||
cfe30b87 DW |
455 | if (__le16_to_cpu(pfn_sb->version_minor) < 1) { |
456 | pfn_sb->start_pad = 0; | |
457 | pfn_sb->end_trunc = 0; | |
458 | } | |
459 | ||
45a0dac0 DW |
460 | if (__le16_to_cpu(pfn_sb->version_minor) < 2) |
461 | pfn_sb->align = 0; | |
462 | ||
e1455744 DW |
463 | switch (le32_to_cpu(pfn_sb->mode)) { |
464 | case PFN_MODE_RAM: | |
e1455744 | 465 | case PFN_MODE_PMEM: |
45eb570a | 466 | break; |
e1455744 DW |
467 | default: |
468 | return -ENXIO; | |
469 | } | |
470 | ||
1ee6667c DW |
471 | align = le32_to_cpu(pfn_sb->align); |
472 | offset = le64_to_cpu(pfn_sb->dataoff); | |
19deaa21 | 473 | start_pad = le32_to_cpu(pfn_sb->start_pad); |
1ee6667c DW |
474 | if (align == 0) |
475 | align = 1UL << ilog2(offset); | |
476 | mode = le32_to_cpu(pfn_sb->mode); | |
477 | ||
e1455744 | 478 | if (!nd_pfn->uuid) { |
1ee6667c DW |
479 | /* |
480 | * When probing a namepace via nd_pfn_probe() the uuid | |
481 | * is NULL (see: nd_pfn_devinit()) we init settings from | |
482 | * pfn_sb | |
483 | */ | |
e1455744 DW |
484 | nd_pfn->uuid = kmemdup(pfn_sb->uuid, 16, GFP_KERNEL); |
485 | if (!nd_pfn->uuid) | |
486 | return -ENOMEM; | |
1ee6667c DW |
487 | nd_pfn->align = align; |
488 | nd_pfn->mode = mode; | |
e1455744 | 489 | } else { |
1ee6667c DW |
490 | /* |
491 | * When probing a pfn / dax instance we validate the | |
492 | * live settings against the pfn_sb | |
493 | */ | |
e1455744 | 494 | if (memcmp(nd_pfn->uuid, pfn_sb->uuid, 16) != 0) |
e5670563 | 495 | return -ENODEV; |
1ee6667c DW |
496 | |
497 | /* | |
498 | * If the uuid validates, but other settings mismatch | |
499 | * return EINVAL because userspace has managed to change | |
500 | * the configuration without specifying new | |
501 | * identification. | |
502 | */ | |
503 | if (nd_pfn->align != align || nd_pfn->mode != mode) { | |
504 | dev_err(&nd_pfn->dev, | |
505 | "init failed, settings mismatch\n"); | |
506 | dev_dbg(&nd_pfn->dev, "align: %lx:%lx mode: %d:%d\n", | |
507 | nd_pfn->align, align, nd_pfn->mode, | |
508 | mode); | |
509 | return -EINVAL; | |
510 | } | |
e1455744 DW |
511 | } |
512 | ||
1ee6667c | 513 | if (align > nvdimm_namespace_capacity(ndns)) { |
315c5625 | 514 | dev_err(&nd_pfn->dev, "alignment: %lx exceeds capacity %llx\n", |
1ee6667c | 515 | align, nvdimm_namespace_capacity(ndns)); |
315c5625 DW |
516 | return -EINVAL; |
517 | } | |
518 | ||
e1455744 DW |
519 | /* |
520 | * These warnings are verbose because they can only trigger in | |
521 | * the case where the physical address alignment of the | |
522 | * namespace has changed since the pfn superblock was | |
523 | * established. | |
524 | */ | |
e1455744 | 525 | nsio = to_nd_namespace_io(&ndns->dev); |
9f1e8cee | 526 | if (offset >= resource_size(&nsio->res)) { |
e1455744 DW |
527 | dev_err(&nd_pfn->dev, "pfn array size exceeds capacity of %s\n", |
528 | dev_name(&ndns->dev)); | |
529 | return -EBUSY; | |
530 | } | |
531 | ||
19deaa21 | 532 | if ((align && !IS_ALIGNED(nsio->res.start + offset + start_pad, align)) |
5e24c9fd | 533 | || !IS_ALIGNED(offset, PAGE_SIZE)) { |
1ee6667c DW |
534 | dev_err(&nd_pfn->dev, |
535 | "bad offset: %#llx dax disabled align: %#lx\n", | |
536 | offset, align); | |
315c5625 DW |
537 | return -ENXIO; |
538 | } | |
539 | ||
48af2f7e | 540 | return nd_pfn_clear_memmap_errors(nd_pfn); |
e1455744 | 541 | } |
32ab0a3f | 542 | EXPORT_SYMBOL(nd_pfn_validate); |
e1455744 | 543 | |
200c79da | 544 | int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns) |
e1455744 DW |
545 | { |
546 | int rc; | |
e1455744 | 547 | struct nd_pfn *nd_pfn; |
bd032943 | 548 | struct device *pfn_dev; |
e1455744 DW |
549 | struct nd_pfn_sb *pfn_sb; |
550 | struct nd_region *nd_region = to_nd_region(ndns->dev.parent); | |
551 | ||
552 | if (ndns->force_raw) | |
553 | return -ENODEV; | |
554 | ||
b3fde74e DW |
555 | switch (ndns->claim_class) { |
556 | case NVDIMM_CCLASS_NONE: | |
557 | case NVDIMM_CCLASS_PFN: | |
558 | break; | |
559 | default: | |
560 | return -ENODEV; | |
561 | } | |
562 | ||
e1455744 | 563 | nvdimm_bus_lock(&ndns->dev); |
cd03412a DW |
564 | nd_pfn = nd_pfn_alloc(nd_region); |
565 | pfn_dev = nd_pfn_devinit(nd_pfn, ndns); | |
e1455744 | 566 | nvdimm_bus_unlock(&ndns->dev); |
bd032943 | 567 | if (!pfn_dev) |
e1455744 | 568 | return -ENOMEM; |
7e3e888d | 569 | pfn_sb = devm_kmalloc(dev, sizeof(*pfn_sb), GFP_KERNEL); |
bd032943 | 570 | nd_pfn = to_nd_pfn(pfn_dev); |
e1455744 | 571 | nd_pfn->pfn_sb = pfn_sb; |
c5ed9268 | 572 | rc = nd_pfn_validate(nd_pfn, PFN_SIG); |
426824d6 | 573 | dev_dbg(dev, "pfn: %s\n", rc == 0 ? dev_name(pfn_dev) : "<none>"); |
e1455744 | 574 | if (rc < 0) { |
452bae0a | 575 | nd_detach_ndns(pfn_dev, &nd_pfn->ndns); |
bd032943 | 576 | put_device(pfn_dev); |
e1455744 | 577 | } else |
bd032943 | 578 | __nd_device_register(pfn_dev); |
e1455744 DW |
579 | |
580 | return rc; | |
581 | } | |
582 | EXPORT_SYMBOL(nd_pfn_probe); | |
ac515c08 | 583 | |
11a35810 DW |
584 | static u32 info_block_reserve(void) |
585 | { | |
586 | return ALIGN(SZ_8K, PAGE_SIZE); | |
587 | } | |
588 | ||
ac515c08 DW |
589 | /* |
590 | * We hotplug memory at section granularity, pad the reserved area from | |
591 | * the previous section base to the namespace base address. | |
592 | */ | |
593 | static unsigned long init_altmap_base(resource_size_t base) | |
594 | { | |
595 | unsigned long base_pfn = PHYS_PFN(base); | |
596 | ||
597 | return PFN_SECTION_ALIGN_DOWN(base_pfn); | |
598 | } | |
599 | ||
600 | static unsigned long init_altmap_reserve(resource_size_t base) | |
601 | { | |
11a35810 | 602 | unsigned long reserve = info_block_reserve() >> PAGE_SHIFT; |
ac515c08 DW |
603 | unsigned long base_pfn = PHYS_PFN(base); |
604 | ||
605 | reserve += base_pfn - PFN_SECTION_ALIGN_DOWN(base_pfn); | |
606 | return reserve; | |
607 | } | |
608 | ||
e8d51348 | 609 | static int __nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap) |
ac515c08 | 610 | { |
e8d51348 CH |
611 | struct resource *res = &pgmap->res; |
612 | struct vmem_altmap *altmap = &pgmap->altmap; | |
ac515c08 DW |
613 | struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; |
614 | u64 offset = le64_to_cpu(pfn_sb->dataoff); | |
615 | u32 start_pad = __le32_to_cpu(pfn_sb->start_pad); | |
616 | u32 end_trunc = __le32_to_cpu(pfn_sb->end_trunc); | |
11a35810 | 617 | u32 reserve = info_block_reserve(); |
ac515c08 DW |
618 | struct nd_namespace_common *ndns = nd_pfn->ndns; |
619 | struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); | |
620 | resource_size_t base = nsio->res.start + start_pad; | |
621 | struct vmem_altmap __altmap = { | |
622 | .base_pfn = init_altmap_base(base), | |
623 | .reserve = init_altmap_reserve(base), | |
624 | }; | |
625 | ||
626 | memcpy(res, &nsio->res, sizeof(*res)); | |
627 | res->start += start_pad; | |
628 | res->end -= end_trunc; | |
629 | ||
ac515c08 | 630 | if (nd_pfn->mode == PFN_MODE_RAM) { |
11a35810 | 631 | if (offset < reserve) |
e8d51348 | 632 | return -EINVAL; |
ac515c08 | 633 | nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns); |
ac515c08 | 634 | } else if (nd_pfn->mode == PFN_MODE_PMEM) { |
d5483fed DW |
635 | nd_pfn->npfns = PFN_SECTION_ALIGN_UP((resource_size(res) |
636 | - offset) / PAGE_SIZE); | |
ac515c08 DW |
637 | if (le64_to_cpu(nd_pfn->pfn_sb->npfns) > nd_pfn->npfns) |
638 | dev_info(&nd_pfn->dev, | |
639 | "number of pfns truncated from %lld to %ld\n", | |
640 | le64_to_cpu(nd_pfn->pfn_sb->npfns), | |
641 | nd_pfn->npfns); | |
642 | memcpy(altmap, &__altmap, sizeof(*altmap)); | |
11a35810 | 643 | altmap->free = PHYS_PFN(offset - reserve); |
ac515c08 | 644 | altmap->alloc = 0; |
514caf23 | 645 | pgmap->flags |= PGMAP_ALTMAP_VALID; |
ac515c08 | 646 | } else |
e8d51348 | 647 | return -ENXIO; |
ac515c08 | 648 | |
e8d51348 | 649 | return 0; |
ac515c08 DW |
650 | } |
651 | ||
41fce90f DW |
652 | static u64 phys_pmem_align_down(struct nd_pfn *nd_pfn, u64 phys) |
653 | { | |
654 | return min_t(u64, PHYS_SECTION_ALIGN_DOWN(phys), | |
655 | ALIGN_DOWN(phys, nd_pfn->align)); | |
656 | } | |
657 | ||
ae86cbfe DW |
658 | /* |
659 | * Check if pmem collides with 'System RAM', or other regions when | |
660 | * section aligned. Trim it accordingly. | |
661 | */ | |
662 | static void trim_pfn_device(struct nd_pfn *nd_pfn, u32 *start_pad, u32 *end_trunc) | |
663 | { | |
664 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
665 | struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); | |
666 | struct nd_region *nd_region = to_nd_region(nd_pfn->dev.parent); | |
667 | const resource_size_t start = nsio->res.start; | |
668 | const resource_size_t end = start + resource_size(&nsio->res); | |
669 | resource_size_t adjust, size; | |
670 | ||
671 | *start_pad = 0; | |
672 | *end_trunc = 0; | |
673 | ||
674 | adjust = start - PHYS_SECTION_ALIGN_DOWN(start); | |
675 | size = resource_size(&nsio->res) + adjust; | |
676 | if (region_intersects(start - adjust, size, IORESOURCE_SYSTEM_RAM, | |
677 | IORES_DESC_NONE) == REGION_MIXED | |
678 | || nd_region_conflict(nd_region, start - adjust, size)) | |
679 | *start_pad = PHYS_SECTION_ALIGN_UP(start) - start; | |
680 | ||
681 | /* Now check that end of the range does not collide. */ | |
682 | adjust = PHYS_SECTION_ALIGN_UP(end) - end; | |
683 | size = resource_size(&nsio->res) + adjust; | |
684 | if (region_intersects(start, size, IORESOURCE_SYSTEM_RAM, | |
685 | IORES_DESC_NONE) == REGION_MIXED | |
686 | || !IS_ALIGNED(end, nd_pfn->align) | |
f101ada7 | 687 | || nd_region_conflict(nd_region, start, size)) |
ae86cbfe DW |
688 | *end_trunc = end - phys_pmem_align_down(nd_pfn, end); |
689 | } | |
690 | ||
ac515c08 DW |
691 | static int nd_pfn_init(struct nd_pfn *nd_pfn) |
692 | { | |
693 | struct nd_namespace_common *ndns = nd_pfn->ndns; | |
ae86cbfe | 694 | struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); |
11a35810 | 695 | u32 start_pad, end_trunc, reserve = info_block_reserve(); |
ac515c08 | 696 | resource_size_t start, size; |
ac515c08 DW |
697 | struct nd_region *nd_region; |
698 | struct nd_pfn_sb *pfn_sb; | |
699 | unsigned long npfns; | |
700 | phys_addr_t offset; | |
c5ed9268 | 701 | const char *sig; |
ac515c08 DW |
702 | u64 checksum; |
703 | int rc; | |
704 | ||
7e3e888d | 705 | pfn_sb = devm_kmalloc(&nd_pfn->dev, sizeof(*pfn_sb), GFP_KERNEL); |
ac515c08 DW |
706 | if (!pfn_sb) |
707 | return -ENOMEM; | |
708 | ||
709 | nd_pfn->pfn_sb = pfn_sb; | |
c5ed9268 DW |
710 | if (is_nd_dax(&nd_pfn->dev)) |
711 | sig = DAX_SIG; | |
712 | else | |
713 | sig = PFN_SIG; | |
7e3e888d | 714 | |
c5ed9268 | 715 | rc = nd_pfn_validate(nd_pfn, sig); |
ac515c08 DW |
716 | if (rc != -ENODEV) |
717 | return rc; | |
718 | ||
719 | /* no info block, do init */; | |
7e3e888d DW |
720 | memset(pfn_sb, 0, sizeof(*pfn_sb)); |
721 | ||
ac515c08 DW |
722 | nd_region = to_nd_region(nd_pfn->dev.parent); |
723 | if (nd_region->ro) { | |
724 | dev_info(&nd_pfn->dev, | |
725 | "%s is read-only, unable to init metadata\n", | |
726 | dev_name(&nd_region->dev)); | |
727 | return -ENXIO; | |
728 | } | |
729 | ||
730 | memset(pfn_sb, 0, sizeof(*pfn_sb)); | |
731 | ||
ae86cbfe | 732 | trim_pfn_device(nd_pfn, &start_pad, &end_trunc); |
ac515c08 | 733 | if (start_pad + end_trunc) |
41fce90f | 734 | dev_info(&nd_pfn->dev, "%s alignment collision, truncate %d bytes\n", |
ac515c08 DW |
735 | dev_name(&ndns->dev), start_pad + end_trunc); |
736 | ||
737 | /* | |
738 | * Note, we use 64 here for the standard size of struct page, | |
739 | * debugging options may cause it to be larger in which case the | |
740 | * implementation will limit the pfns advertised through | |
741 | * ->direct_access() to those that are included in the memmap. | |
742 | */ | |
ae86cbfe | 743 | start = nsio->res.start + start_pad; |
ac515c08 | 744 | size = resource_size(&nsio->res); |
11a35810 | 745 | npfns = PFN_SECTION_ALIGN_UP((size - start_pad - end_trunc - reserve) |
d5483fed | 746 | / PAGE_SIZE); |
594d6d96 | 747 | if (nd_pfn->mode == PFN_MODE_PMEM) { |
594d6d96 | 748 | /* |
0dd69643 OH |
749 | * The altmap should be padded out to the block size used |
750 | * when populating the vmemmap. This *should* be equal to | |
751 | * PMD_SIZE for most architectures. | |
594d6d96 | 752 | */ |
4960461f | 753 | offset = ALIGN(start + reserve + 64 * npfns, |
0dd69643 | 754 | max(nd_pfn->align, PMD_SIZE)) - start; |
594d6d96 | 755 | } else if (nd_pfn->mode == PFN_MODE_RAM) |
4960461f | 756 | offset = ALIGN(start + reserve, nd_pfn->align) - start; |
ac515c08 DW |
757 | else |
758 | return -ENXIO; | |
759 | ||
760 | if (offset + start_pad + end_trunc >= size) { | |
761 | dev_err(&nd_pfn->dev, "%s unable to satisfy requested alignment\n", | |
762 | dev_name(&ndns->dev)); | |
763 | return -ENXIO; | |
764 | } | |
765 | ||
766 | npfns = (size - offset - start_pad - end_trunc) / SZ_4K; | |
767 | pfn_sb->mode = cpu_to_le32(nd_pfn->mode); | |
768 | pfn_sb->dataoff = cpu_to_le64(offset); | |
769 | pfn_sb->npfns = cpu_to_le64(npfns); | |
c5ed9268 | 770 | memcpy(pfn_sb->signature, sig, PFN_SIG_LEN); |
ac515c08 DW |
771 | memcpy(pfn_sb->uuid, nd_pfn->uuid, 16); |
772 | memcpy(pfn_sb->parent_uuid, nd_dev_to_uuid(&ndns->dev), 16); | |
773 | pfn_sb->version_major = cpu_to_le16(1); | |
7e3e888d | 774 | pfn_sb->version_minor = cpu_to_le16(3); |
ac515c08 DW |
775 | pfn_sb->start_pad = cpu_to_le32(start_pad); |
776 | pfn_sb->end_trunc = cpu_to_le32(end_trunc); | |
45a0dac0 | 777 | pfn_sb->align = cpu_to_le32(nd_pfn->align); |
ac515c08 DW |
778 | checksum = nd_sb_checksum((struct nd_gen_sb *) pfn_sb); |
779 | pfn_sb->checksum = cpu_to_le64(checksum); | |
780 | ||
3ae3d67b | 781 | return nvdimm_write_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb), 0); |
ac515c08 DW |
782 | } |
783 | ||
784 | /* | |
785 | * Determine the effective resource range and vmem_altmap from an nd_pfn | |
786 | * instance. | |
787 | */ | |
e8d51348 | 788 | int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap) |
ac515c08 DW |
789 | { |
790 | int rc; | |
791 | ||
792 | if (!nd_pfn->uuid || !nd_pfn->ndns) | |
e8d51348 | 793 | return -ENODEV; |
ac515c08 DW |
794 | |
795 | rc = nd_pfn_init(nd_pfn); | |
796 | if (rc) | |
e8d51348 | 797 | return rc; |
ac515c08 | 798 | |
e8d51348 CH |
799 | /* we need a valid pfn_sb before we can init a dev_pagemap */ |
800 | return __nvdimm_setup_pfn(nd_pfn, pgmap); | |
ac515c08 DW |
801 | } |
802 | EXPORT_SYMBOL_GPL(nvdimm_setup_pfn); |