]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
09d4e0ed PM |
2 | /* |
3 | * Generic implementation of 64-bit atomics using spinlocks, | |
4 | * useful on processors that don't have 64-bit atomic instructions. | |
5 | * | |
6 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
09d4e0ed PM |
7 | */ |
8 | #include <linux/types.h> | |
9 | #include <linux/cache.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/init.h> | |
8bc3bcc9 | 12 | #include <linux/export.h> |
60063497 | 13 | #include <linux/atomic.h> |
09d4e0ed PM |
14 | |
15 | /* | |
16 | * We use a hashed array of spinlocks to provide exclusive access | |
17 | * to each atomic64_t variable. Since this is expected to used on | |
18 | * systems with small numbers of CPUs (<= 4 or so), we use a | |
19 | * relatively small array of 16 spinlocks to avoid wasting too much | |
20 | * memory on the spinlock array. | |
21 | */ | |
22 | #define NR_LOCKS 16 | |
23 | ||
24 | /* | |
25 | * Ensure each lock is in a separate cacheline. | |
26 | */ | |
27 | static union { | |
f59ca058 | 28 | raw_spinlock_t lock; |
09d4e0ed | 29 | char pad[L1_CACHE_BYTES]; |
fcc16882 SB |
30 | } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = { |
31 | [0 ... (NR_LOCKS - 1)] = { | |
32 | .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock), | |
33 | }, | |
34 | }; | |
09d4e0ed | 35 | |
cb475de3 | 36 | static inline raw_spinlock_t *lock_addr(const atomic64_t *v) |
09d4e0ed PM |
37 | { |
38 | unsigned long addr = (unsigned long) v; | |
39 | ||
40 | addr >>= L1_CACHE_SHIFT; | |
41 | addr ^= (addr >> 8) ^ (addr >> 16); | |
42 | return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; | |
43 | } | |
44 | ||
45 | long long atomic64_read(const atomic64_t *v) | |
46 | { | |
47 | unsigned long flags; | |
cb475de3 | 48 | raw_spinlock_t *lock = lock_addr(v); |
09d4e0ed PM |
49 | long long val; |
50 | ||
f59ca058 | 51 | raw_spin_lock_irqsave(lock, flags); |
09d4e0ed | 52 | val = v->counter; |
f59ca058 | 53 | raw_spin_unlock_irqrestore(lock, flags); |
09d4e0ed PM |
54 | return val; |
55 | } | |
3fc7b4b2 | 56 | EXPORT_SYMBOL(atomic64_read); |
09d4e0ed PM |
57 | |
58 | void atomic64_set(atomic64_t *v, long long i) | |
59 | { | |
60 | unsigned long flags; | |
cb475de3 | 61 | raw_spinlock_t *lock = lock_addr(v); |
09d4e0ed | 62 | |
f59ca058 | 63 | raw_spin_lock_irqsave(lock, flags); |
09d4e0ed | 64 | v->counter = i; |
f59ca058 | 65 | raw_spin_unlock_irqrestore(lock, flags); |
09d4e0ed | 66 | } |
3fc7b4b2 | 67 | EXPORT_SYMBOL(atomic64_set); |
09d4e0ed | 68 | |
560cb12a PZ |
69 | #define ATOMIC64_OP(op, c_op) \ |
70 | void atomic64_##op(long long a, atomic64_t *v) \ | |
71 | { \ | |
72 | unsigned long flags; \ | |
73 | raw_spinlock_t *lock = lock_addr(v); \ | |
74 | \ | |
75 | raw_spin_lock_irqsave(lock, flags); \ | |
76 | v->counter c_op a; \ | |
77 | raw_spin_unlock_irqrestore(lock, flags); \ | |
78 | } \ | |
79 | EXPORT_SYMBOL(atomic64_##op); | |
80 | ||
81 | #define ATOMIC64_OP_RETURN(op, c_op) \ | |
82 | long long atomic64_##op##_return(long long a, atomic64_t *v) \ | |
83 | { \ | |
84 | unsigned long flags; \ | |
85 | raw_spinlock_t *lock = lock_addr(v); \ | |
86 | long long val; \ | |
87 | \ | |
88 | raw_spin_lock_irqsave(lock, flags); \ | |
89 | val = (v->counter c_op a); \ | |
90 | raw_spin_unlock_irqrestore(lock, flags); \ | |
91 | return val; \ | |
92 | } \ | |
93 | EXPORT_SYMBOL(atomic64_##op##_return); | |
94 | ||
28aa2bda PZ |
95 | #define ATOMIC64_FETCH_OP(op, c_op) \ |
96 | long long atomic64_fetch_##op(long long a, atomic64_t *v) \ | |
97 | { \ | |
98 | unsigned long flags; \ | |
99 | raw_spinlock_t *lock = lock_addr(v); \ | |
100 | long long val; \ | |
101 | \ | |
102 | raw_spin_lock_irqsave(lock, flags); \ | |
103 | val = v->counter; \ | |
104 | v->counter c_op a; \ | |
105 | raw_spin_unlock_irqrestore(lock, flags); \ | |
106 | return val; \ | |
107 | } \ | |
108 | EXPORT_SYMBOL(atomic64_fetch_##op); | |
109 | ||
560cb12a PZ |
110 | #define ATOMIC64_OPS(op, c_op) \ |
111 | ATOMIC64_OP(op, c_op) \ | |
28aa2bda PZ |
112 | ATOMIC64_OP_RETURN(op, c_op) \ |
113 | ATOMIC64_FETCH_OP(op, c_op) | |
560cb12a PZ |
114 | |
115 | ATOMIC64_OPS(add, +=) | |
116 | ATOMIC64_OPS(sub, -=) | |
117 | ||
118 | #undef ATOMIC64_OPS | |
28aa2bda PZ |
119 | #define ATOMIC64_OPS(op, c_op) \ |
120 | ATOMIC64_OP(op, c_op) \ | |
121 | ATOMIC64_OP_RETURN(op, c_op) \ | |
122 | ATOMIC64_FETCH_OP(op, c_op) | |
123 | ||
124 | ATOMIC64_OPS(and, &=) | |
125 | ATOMIC64_OPS(or, |=) | |
126 | ATOMIC64_OPS(xor, ^=) | |
127 | ||
128 | #undef ATOMIC64_OPS | |
129 | #undef ATOMIC64_FETCH_OP | |
560cb12a PZ |
130 | #undef ATOMIC64_OP_RETURN |
131 | #undef ATOMIC64_OP | |
09d4e0ed PM |
132 | |
133 | long long atomic64_dec_if_positive(atomic64_t *v) | |
134 | { | |
135 | unsigned long flags; | |
cb475de3 | 136 | raw_spinlock_t *lock = lock_addr(v); |
09d4e0ed PM |
137 | long long val; |
138 | ||
f59ca058 | 139 | raw_spin_lock_irqsave(lock, flags); |
09d4e0ed PM |
140 | val = v->counter - 1; |
141 | if (val >= 0) | |
142 | v->counter = val; | |
f59ca058 | 143 | raw_spin_unlock_irqrestore(lock, flags); |
09d4e0ed PM |
144 | return val; |
145 | } | |
3fc7b4b2 | 146 | EXPORT_SYMBOL(atomic64_dec_if_positive); |
09d4e0ed PM |
147 | |
148 | long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) | |
149 | { | |
150 | unsigned long flags; | |
cb475de3 | 151 | raw_spinlock_t *lock = lock_addr(v); |
09d4e0ed PM |
152 | long long val; |
153 | ||
f59ca058 | 154 | raw_spin_lock_irqsave(lock, flags); |
09d4e0ed PM |
155 | val = v->counter; |
156 | if (val == o) | |
157 | v->counter = n; | |
f59ca058 | 158 | raw_spin_unlock_irqrestore(lock, flags); |
09d4e0ed PM |
159 | return val; |
160 | } | |
3fc7b4b2 | 161 | EXPORT_SYMBOL(atomic64_cmpxchg); |
09d4e0ed PM |
162 | |
163 | long long atomic64_xchg(atomic64_t *v, long long new) | |
164 | { | |
165 | unsigned long flags; | |
cb475de3 | 166 | raw_spinlock_t *lock = lock_addr(v); |
09d4e0ed PM |
167 | long long val; |
168 | ||
f59ca058 | 169 | raw_spin_lock_irqsave(lock, flags); |
09d4e0ed PM |
170 | val = v->counter; |
171 | v->counter = new; | |
f59ca058 | 172 | raw_spin_unlock_irqrestore(lock, flags); |
09d4e0ed PM |
173 | return val; |
174 | } | |
3fc7b4b2 | 175 | EXPORT_SYMBOL(atomic64_xchg); |
09d4e0ed | 176 | |
00b808ab | 177 | long long atomic64_fetch_add_unless(atomic64_t *v, long long a, long long u) |
09d4e0ed PM |
178 | { |
179 | unsigned long flags; | |
cb475de3 | 180 | raw_spinlock_t *lock = lock_addr(v); |
00b808ab | 181 | long long val; |
09d4e0ed | 182 | |
f59ca058 | 183 | raw_spin_lock_irqsave(lock, flags); |
00b808ab MR |
184 | val = v->counter; |
185 | if (val != u) | |
09d4e0ed | 186 | v->counter += a; |
f59ca058 | 187 | raw_spin_unlock_irqrestore(lock, flags); |
00b808ab MR |
188 | |
189 | return val; | |
09d4e0ed | 190 | } |
00b808ab | 191 | EXPORT_SYMBOL(atomic64_fetch_add_unless); |