projects / thirdparty / kernel / linux.git / blame
| Commit | Line | Data |
|---|---|---|
| b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 418b2977 LR |
2 | #ifndef PERCPU_H |
| 3 | #define PERCPU_H | |
| 4 | ||
| 5 | #include <stddef.h> | |
| 6 | #include "bug_on.h" | |
| 7 | #include "preempt.h" | |
| 8 | ||
| 9 | #define __percpu | |
| 10 | ||
| 11 | /* Maximum size of any percpu data. */ | |
| 12 | #define PERCPU_OFFSET (4 * sizeof(long)) | |
| 13 | ||
| 14 | /* Ignore alignment, as CBMC doesn't care about false sharing. */ | |
| 15 | #define alloc_percpu(type) __alloc_percpu(sizeof(type), 1) | |
| 16 | ||
| 17 | static inline void *__alloc_percpu(size_t size, size_t align) | |
| 18 | { | |
| 19 | BUG(); | |
| 20 | return NULL; | |
| 21 | } | |
| 22 | ||
| 23 | static inline void free_percpu(void *ptr) | |
| 24 | { | |
| 25 | BUG(); | |
| 26 | } | |
| 27 | ||
| 28 | #define per_cpu_ptr(ptr, cpu) \ | |
| 29 | ((typeof(ptr)) ((char *) (ptr) + PERCPU_OFFSET * cpu)) | |
| 30 | ||
| 31 | #define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1) | |
| 32 | #define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1) | |
| 33 | #define __this_cpu_sub(pcp, n) __this_cpu_add(pcp, -(typeof(pcp)) (n)) | |
| 34 | ||
| 35 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1) | |
| 36 | #define this_cpu_dec(pcp) this_cpu_sub(pcp, 1) | |
| 37 | #define this_cpu_sub(pcp, n) this_cpu_add(pcp, -(typeof(pcp)) (n)) | |
| 38 | ||
| 39 | /* Make CBMC use atomics to work around bug. */ | |
| 40 | #ifdef RUN | |
| 41 | #define THIS_CPU_ADD_HELPER(ptr, x) (*(ptr) += (x)) | |
| 42 | #else | |
| 43 | /* | |
| 44 | * Split the atomic into a read and a write so that it has the least | |
| 45 | * possible ordering. | |
| 46 | */ | |
| 47 | #define THIS_CPU_ADD_HELPER(ptr, x) \ | |
| 48 | do { \ | |
| 49 | typeof(ptr) this_cpu_add_helper_ptr = (ptr); \ | |
| 50 | typeof(ptr) this_cpu_add_helper_x = (x); \ | |
| 51 | typeof(*ptr) this_cpu_add_helper_temp; \ | |
| 52 | __CPROVER_atomic_begin(); \ | |
| 53 | this_cpu_add_helper_temp = *(this_cpu_add_helper_ptr); \ | |
| 54 | __CPROVER_atomic_end(); \ | |
| 55 | this_cpu_add_helper_temp += this_cpu_add_helper_x; \ | |
| 56 | __CPROVER_atomic_begin(); \ | |
| 57 | *(this_cpu_add_helper_ptr) = this_cpu_add_helper_temp; \ | |
| 58 | __CPROVER_atomic_end(); \ | |
| 59 | } while (0) | |
| 60 | #endif | |
| 61 | ||
| 62 | /* | |
| 63 | * For some reason CBMC needs an atomic operation even though this is percpu | |
| 64 | * data. | |
| 65 | */ | |
| 66 | #define __this_cpu_add(pcp, n) \ | |
| 67 | do { \ | |
| 68 | BUG_ON(preemptible()); \ | |
| 69 | THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), thread_cpu_id), \ | |
| 70 | (typeof(pcp)) (n)); \ | |
| 71 | } while (0) | |
| 72 | ||
| 73 | #define this_cpu_add(pcp, n) \ | |
| 74 | do { \ | |
| 75 | int this_cpu_add_impl_cpu = get_cpu(); \ | |
| 76 | THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), this_cpu_add_impl_cpu), \ | |
| 77 | (typeof(pcp)) (n)); \ | |
| 78 | put_cpu(); \ | |
| 79 | } while (0) | |
| 80 | ||
| 81 | /* | |
| 82 | * This will cause a compiler warning because of the cast from char[][] to | |
| 83 | * type*. This will cause a compile time error if type is too big. | |
| 84 | */ | |
| 85 | #define DEFINE_PER_CPU(type, name) \ | |
| 86 | char name[NR_CPUS][PERCPU_OFFSET]; \ | |
| 87 | typedef char percpu_too_big_##name \ | |
| 88 | [sizeof(type) > PERCPU_OFFSET ? -1 : 1] | |
| 89 | ||
| 90 | #define for_each_possible_cpu(cpu) \ | |
| 91 | for ((cpu) = 0; (cpu) < NR_CPUS; ++(cpu)) | |
| 92 | ||
| 93 | #endif |