]>
git.ipfire.org Git - people/ms/linux.git/blob - fs/pstore/ram_core.c
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 Google, Inc.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/device.h>
10 #include <linux/errno.h>
11 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/list.h>
15 #include <linux/memblock.h>
16 #include <linux/pstore_ram.h>
17 #include <linux/rslib.h>
18 #include <linux/slab.h>
19 #include <linux/uaccess.h>
20 #include <linux/vmalloc.h>
24 * struct persistent_ram_buffer - persistent circular RAM buffer
27 * signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value)
29 * offset into @data where the beginning of the stored bytes begin
31 * number of valid bytes stored in @data
33 struct persistent_ram_buffer
{
40 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
42 static inline size_t buffer_size(struct persistent_ram_zone
*prz
)
44 return atomic_read(&prz
->buffer
->size
);
47 static inline size_t buffer_start(struct persistent_ram_zone
*prz
)
49 return atomic_read(&prz
->buffer
->start
);
52 /* increase and wrap the start pointer, returning the old value */
53 static size_t buffer_start_add(struct persistent_ram_zone
*prz
, size_t a
)
57 unsigned long flags
= 0;
59 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
60 raw_spin_lock_irqsave(&prz
->buffer_lock
, flags
);
62 old
= atomic_read(&prz
->buffer
->start
);
64 while (unlikely(new >= prz
->buffer_size
))
65 new -= prz
->buffer_size
;
66 atomic_set(&prz
->buffer
->start
, new);
68 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
69 raw_spin_unlock_irqrestore(&prz
->buffer_lock
, flags
);
74 /* increase the size counter until it hits the max size */
75 static void buffer_size_add(struct persistent_ram_zone
*prz
, size_t a
)
79 unsigned long flags
= 0;
81 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
82 raw_spin_lock_irqsave(&prz
->buffer_lock
, flags
);
84 old
= atomic_read(&prz
->buffer
->size
);
85 if (old
== prz
->buffer_size
)
89 if (new > prz
->buffer_size
)
90 new = prz
->buffer_size
;
91 atomic_set(&prz
->buffer
->size
, new);
94 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
95 raw_spin_unlock_irqrestore(&prz
->buffer_lock
, flags
);
98 static void notrace
persistent_ram_encode_rs8(struct persistent_ram_zone
*prz
,
99 uint8_t *data
, size_t len
, uint8_t *ecc
)
103 /* Initialize the parity buffer */
104 memset(prz
->ecc_info
.par
, 0,
105 prz
->ecc_info
.ecc_size
* sizeof(prz
->ecc_info
.par
[0]));
106 encode_rs8(prz
->rs_decoder
, data
, len
, prz
->ecc_info
.par
, 0);
107 for (i
= 0; i
< prz
->ecc_info
.ecc_size
; i
++)
108 ecc
[i
] = prz
->ecc_info
.par
[i
];
111 static int persistent_ram_decode_rs8(struct persistent_ram_zone
*prz
,
112 void *data
, size_t len
, uint8_t *ecc
)
116 for (i
= 0; i
< prz
->ecc_info
.ecc_size
; i
++)
117 prz
->ecc_info
.par
[i
] = ecc
[i
];
118 return decode_rs8(prz
->rs_decoder
, data
, prz
->ecc_info
.par
, len
,
119 NULL
, 0, NULL
, 0, NULL
);
122 static void notrace
persistent_ram_update_ecc(struct persistent_ram_zone
*prz
,
123 unsigned int start
, unsigned int count
)
125 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
126 uint8_t *buffer_end
= buffer
->data
+ prz
->buffer_size
;
129 int ecc_block_size
= prz
->ecc_info
.block_size
;
130 int ecc_size
= prz
->ecc_info
.ecc_size
;
131 int size
= ecc_block_size
;
136 block
= buffer
->data
+ (start
& ~(ecc_block_size
- 1));
137 par
= prz
->par_buffer
+ (start
/ ecc_block_size
) * ecc_size
;
140 if (block
+ ecc_block_size
> buffer_end
)
141 size
= buffer_end
- block
;
142 persistent_ram_encode_rs8(prz
, block
, size
, par
);
143 block
+= ecc_block_size
;
145 } while (block
< buffer
->data
+ start
+ count
);
148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone
*prz
)
150 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
152 if (!prz
->ecc_info
.ecc_size
)
155 persistent_ram_encode_rs8(prz
, (uint8_t *)buffer
, sizeof(*buffer
),
159 static void persistent_ram_ecc_old(struct persistent_ram_zone
*prz
)
161 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
165 if (!prz
->ecc_info
.ecc_size
)
168 block
= buffer
->data
;
169 par
= prz
->par_buffer
;
170 while (block
< buffer
->data
+ buffer_size(prz
)) {
172 int size
= prz
->ecc_info
.block_size
;
173 if (block
+ size
> buffer
->data
+ prz
->buffer_size
)
174 size
= buffer
->data
+ prz
->buffer_size
- block
;
175 numerr
= persistent_ram_decode_rs8(prz
, block
, size
, par
);
177 pr_devel("error in block %p, %d\n", block
, numerr
);
178 prz
->corrected_bytes
+= numerr
;
179 } else if (numerr
< 0) {
180 pr_devel("uncorrectable error in block %p\n", block
);
183 block
+= prz
->ecc_info
.block_size
;
184 par
+= prz
->ecc_info
.ecc_size
;
188 static int persistent_ram_init_ecc(struct persistent_ram_zone
*prz
,
189 struct persistent_ram_ecc_info
*ecc_info
)
192 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
196 if (!ecc_info
|| !ecc_info
->ecc_size
)
199 prz
->ecc_info
.block_size
= ecc_info
->block_size
?: 128;
200 prz
->ecc_info
.ecc_size
= ecc_info
->ecc_size
?: 16;
201 prz
->ecc_info
.symsize
= ecc_info
->symsize
?: 8;
202 prz
->ecc_info
.poly
= ecc_info
->poly
?: 0x11d;
204 ecc_blocks
= DIV_ROUND_UP(prz
->buffer_size
- prz
->ecc_info
.ecc_size
,
205 prz
->ecc_info
.block_size
+
206 prz
->ecc_info
.ecc_size
);
207 ecc_total
= (ecc_blocks
+ 1) * prz
->ecc_info
.ecc_size
;
208 if (ecc_total
>= prz
->buffer_size
) {
209 pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n",
210 __func__
, prz
->ecc_info
.ecc_size
,
211 ecc_total
, prz
->buffer_size
);
215 prz
->buffer_size
-= ecc_total
;
216 prz
->par_buffer
= buffer
->data
+ prz
->buffer_size
;
217 prz
->par_header
= prz
->par_buffer
+
218 ecc_blocks
* prz
->ecc_info
.ecc_size
;
221 * first consecutive root is 0
222 * primitive element to generate roots = 1
224 prz
->rs_decoder
= init_rs(prz
->ecc_info
.symsize
, prz
->ecc_info
.poly
,
225 0, 1, prz
->ecc_info
.ecc_size
);
226 if (prz
->rs_decoder
== NULL
) {
227 pr_info("init_rs failed\n");
231 /* allocate workspace instead of using stack VLA */
232 prz
->ecc_info
.par
= kmalloc_array(prz
->ecc_info
.ecc_size
,
233 sizeof(*prz
->ecc_info
.par
),
235 if (!prz
->ecc_info
.par
) {
236 pr_err("cannot allocate ECC parity workspace\n");
240 prz
->corrected_bytes
= 0;
243 numerr
= persistent_ram_decode_rs8(prz
, buffer
, sizeof(*buffer
),
246 pr_info("error in header, %d\n", numerr
);
247 prz
->corrected_bytes
+= numerr
;
248 } else if (numerr
< 0) {
249 pr_info_ratelimited("uncorrectable error in header\n");
256 ssize_t
persistent_ram_ecc_string(struct persistent_ram_zone
*prz
,
257 char *str
, size_t len
)
261 if (!prz
->ecc_info
.ecc_size
)
264 if (prz
->corrected_bytes
|| prz
->bad_blocks
)
265 ret
= snprintf(str
, len
, ""
266 "\nECC: %d Corrected bytes, %d unrecoverable blocks\n",
267 prz
->corrected_bytes
, prz
->bad_blocks
);
269 ret
= snprintf(str
, len
, "\nECC: No errors detected\n");
274 static void notrace
persistent_ram_update(struct persistent_ram_zone
*prz
,
275 const void *s
, unsigned int start
, unsigned int count
)
277 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
278 memcpy_toio(buffer
->data
+ start
, s
, count
);
279 persistent_ram_update_ecc(prz
, start
, count
);
282 static int notrace
persistent_ram_update_user(struct persistent_ram_zone
*prz
,
283 const void __user
*s
, unsigned int start
, unsigned int count
)
285 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
286 int ret
= unlikely(copy_from_user(buffer
->data
+ start
, s
, count
)) ?
288 persistent_ram_update_ecc(prz
, start
, count
);
292 void persistent_ram_save_old(struct persistent_ram_zone
*prz
)
294 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
295 size_t size
= buffer_size(prz
);
296 size_t start
= buffer_start(prz
);
302 persistent_ram_ecc_old(prz
);
303 prz
->old_log
= kmalloc(size
, GFP_KERNEL
);
306 pr_err("failed to allocate buffer\n");
310 prz
->old_log_size
= size
;
311 memcpy_fromio(prz
->old_log
, &buffer
->data
[start
], size
- start
);
312 memcpy_fromio(prz
->old_log
+ size
- start
, &buffer
->data
[0], start
);
315 int notrace
persistent_ram_write(struct persistent_ram_zone
*prz
,
316 const void *s
, unsigned int count
)
322 if (unlikely(c
> prz
->buffer_size
)) {
323 s
+= c
- prz
->buffer_size
;
324 c
= prz
->buffer_size
;
327 buffer_size_add(prz
, c
);
329 start
= buffer_start_add(prz
, c
);
331 rem
= prz
->buffer_size
- start
;
332 if (unlikely(rem
< c
)) {
333 persistent_ram_update(prz
, s
, start
, rem
);
338 persistent_ram_update(prz
, s
, start
, c
);
340 persistent_ram_update_header_ecc(prz
);
345 int notrace
persistent_ram_write_user(struct persistent_ram_zone
*prz
,
346 const void __user
*s
, unsigned int count
)
348 int rem
, ret
= 0, c
= count
;
351 if (unlikely(c
> prz
->buffer_size
)) {
352 s
+= c
- prz
->buffer_size
;
353 c
= prz
->buffer_size
;
356 buffer_size_add(prz
, c
);
358 start
= buffer_start_add(prz
, c
);
360 rem
= prz
->buffer_size
- start
;
361 if (unlikely(rem
< c
)) {
362 ret
= persistent_ram_update_user(prz
, s
, start
, rem
);
368 ret
= persistent_ram_update_user(prz
, s
, start
, c
);
370 persistent_ram_update_header_ecc(prz
);
372 return unlikely(ret
) ? ret
: count
;
375 size_t persistent_ram_old_size(struct persistent_ram_zone
*prz
)
377 return prz
->old_log_size
;
380 void *persistent_ram_old(struct persistent_ram_zone
*prz
)
385 void persistent_ram_free_old(struct persistent_ram_zone
*prz
)
389 prz
->old_log_size
= 0;
392 void persistent_ram_zap(struct persistent_ram_zone
*prz
)
394 atomic_set(&prz
->buffer
->start
, 0);
395 atomic_set(&prz
->buffer
->size
, 0);
396 persistent_ram_update_header_ecc(prz
);
399 #define MEM_TYPE_WCOMBINE 0
400 #define MEM_TYPE_NONCACHED 1
401 #define MEM_TYPE_NORMAL 2
403 static void *persistent_ram_vmap(phys_addr_t start
, size_t size
,
404 unsigned int memtype
)
407 phys_addr_t page_start
;
408 unsigned int page_count
;
413 page_start
= start
- offset_in_page(start
);
414 page_count
= DIV_ROUND_UP(size
+ offset_in_page(start
), PAGE_SIZE
);
417 case MEM_TYPE_NORMAL
:
420 case MEM_TYPE_NONCACHED
:
421 prot
= pgprot_noncached(PAGE_KERNEL
);
423 case MEM_TYPE_WCOMBINE
:
424 prot
= pgprot_writecombine(PAGE_KERNEL
);
427 pr_err("invalid mem_type=%d\n", memtype
);
431 pages
= kmalloc_array(page_count
, sizeof(struct page
*), GFP_KERNEL
);
433 pr_err("%s: Failed to allocate array for %u pages\n",
434 __func__
, page_count
);
438 for (i
= 0; i
< page_count
; i
++) {
439 phys_addr_t addr
= page_start
+ i
* PAGE_SIZE
;
440 pages
[i
] = pfn_to_page(addr
>> PAGE_SHIFT
);
442 vaddr
= vmap(pages
, page_count
, VM_MAP
, prot
);
446 * Since vmap() uses page granularity, we must add the offset
447 * into the page here, to get the byte granularity address
448 * into the mapping to represent the actual "start" location.
450 return vaddr
+ offset_in_page(start
);
453 static void *persistent_ram_iomap(phys_addr_t start
, size_t size
,
454 unsigned int memtype
, char *label
)
458 if (!request_mem_region(start
, size
, label
?: "ramoops")) {
459 pr_err("request mem region (%s 0x%llx@0x%llx) failed\n",
461 (unsigned long long)size
, (unsigned long long)start
);
466 va
= ioremap(start
, size
);
468 va
= ioremap_wc(start
, size
);
471 * Since request_mem_region() and ioremap() are byte-granularity
472 * there is no need handle anything special like we do when the
473 * vmap() case in persistent_ram_vmap() above.
478 static int persistent_ram_buffer_map(phys_addr_t start
, phys_addr_t size
,
479 struct persistent_ram_zone
*prz
, int memtype
)
484 if (pfn_valid(start
>> PAGE_SHIFT
))
485 prz
->vaddr
= persistent_ram_vmap(start
, size
, memtype
);
487 prz
->vaddr
= persistent_ram_iomap(start
, size
, memtype
,
491 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__
,
492 (unsigned long long)size
, (unsigned long long)start
);
496 prz
->buffer
= prz
->vaddr
;
497 prz
->buffer_size
= size
- sizeof(struct persistent_ram_buffer
);
502 static int persistent_ram_post_init(struct persistent_ram_zone
*prz
, u32 sig
,
503 struct persistent_ram_ecc_info
*ecc_info
)
506 bool zap
= !!(prz
->flags
& PRZ_FLAG_ZAP_OLD
);
508 ret
= persistent_ram_init_ecc(prz
, ecc_info
);
510 pr_warn("ECC failed %s\n", prz
->label
);
514 sig
^= PERSISTENT_RAM_SIG
;
516 if (prz
->buffer
->sig
== sig
) {
517 if (buffer_size(prz
) == 0) {
518 pr_debug("found existing empty buffer\n");
522 if (buffer_size(prz
) > prz
->buffer_size
||
523 buffer_start(prz
) > buffer_size(prz
)) {
524 pr_info("found existing invalid buffer, size %zu, start %zu\n",
525 buffer_size(prz
), buffer_start(prz
));
528 pr_debug("found existing buffer, size %zu, start %zu\n",
529 buffer_size(prz
), buffer_start(prz
));
530 persistent_ram_save_old(prz
);
533 pr_debug("no valid data in buffer (sig = 0x%08x)\n",
535 prz
->buffer
->sig
= sig
;
539 /* Reset missing, invalid, or single-use memory area. */
541 persistent_ram_zap(prz
);
546 void persistent_ram_free(struct persistent_ram_zone
*prz
)
552 if (pfn_valid(prz
->paddr
>> PAGE_SHIFT
)) {
553 /* We must vunmap() at page-granularity. */
554 vunmap(prz
->vaddr
- offset_in_page(prz
->paddr
));
557 release_mem_region(prz
->paddr
, prz
->size
);
561 if (prz
->rs_decoder
) {
562 free_rs(prz
->rs_decoder
);
563 prz
->rs_decoder
= NULL
;
565 kfree(prz
->ecc_info
.par
);
566 prz
->ecc_info
.par
= NULL
;
568 persistent_ram_free_old(prz
);
573 struct persistent_ram_zone
*persistent_ram_new(phys_addr_t start
, size_t size
,
574 u32 sig
, struct persistent_ram_ecc_info
*ecc_info
,
575 unsigned int memtype
, u32 flags
, char *label
)
577 struct persistent_ram_zone
*prz
;
580 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
582 pr_err("failed to allocate persistent ram zone\n");
586 /* Initialize general buffer state. */
587 raw_spin_lock_init(&prz
->buffer_lock
);
589 prz
->label
= kstrdup(label
, GFP_KERNEL
);
591 ret
= persistent_ram_buffer_map(start
, size
, prz
, memtype
);
595 ret
= persistent_ram_post_init(prz
, sig
, ecc_info
);
599 pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n",
600 prz
->label
, prz
->size
, (unsigned long long)prz
->paddr
,
601 sizeof(*prz
->buffer
), prz
->buffer_size
,
602 prz
->size
- sizeof(*prz
->buffer
) - prz
->buffer_size
,
603 prz
->ecc_info
.ecc_size
, prz
->ecc_info
.block_size
);
607 persistent_ram_free(prz
);