]>
Commit | Line | Data |
---|---|---|
f35db108 WM |
1 | //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// |
2 | // | |
3 | // This file is distributed under the University of Illinois Open Source | |
4 | // License. See LICENSE.TXT for details. | |
5 | // | |
6 | //===----------------------------------------------------------------------===// | |
7 | // | |
8 | // This file is a part of ThreadSanitizer/AddressSanitizer runtime. | |
9 | // Not intended for direct inclusion. Include sanitizer_atomic.h. | |
10 | // | |
11 | //===----------------------------------------------------------------------===// | |
12 | ||
13 | #ifndef SANITIZER_ATOMIC_MSVC_H | |
14 | #define SANITIZER_ATOMIC_MSVC_H | |
15 | ||
16 | extern "C" void _ReadWriteBarrier(); | |
17 | #pragma intrinsic(_ReadWriteBarrier) | |
18 | extern "C" void _mm_mfence(); | |
19 | #pragma intrinsic(_mm_mfence) | |
20 | extern "C" void _mm_pause(); | |
21 | #pragma intrinsic(_mm_pause) | |
696d846a MO |
22 | extern "C" char _InterlockedExchange8( // NOLINT |
23 | char volatile *Addend, char Value); // NOLINT | |
24 | #pragma intrinsic(_InterlockedExchange8) | |
25 | extern "C" short _InterlockedExchange16( // NOLINT | |
26 | short volatile *Addend, short Value); // NOLINT | |
27 | #pragma intrinsic(_InterlockedExchange16) | |
28 | extern "C" long _InterlockedExchange( // NOLINT | |
29 | long volatile *Addend, long Value); // NOLINT | |
30 | #pragma intrinsic(_InterlockedExchange) | |
f35db108 WM |
31 | extern "C" long _InterlockedExchangeAdd( // NOLINT |
32 | long volatile * Addend, long Value); // NOLINT | |
33 | #pragma intrinsic(_InterlockedExchangeAdd) | |
10189819 MO |
34 | extern "C" char _InterlockedCompareExchange8( // NOLINT |
35 | char volatile *Destination, // NOLINT | |
36 | char Exchange, char Comparand); // NOLINT | |
37 | #pragma intrinsic(_InterlockedCompareExchange8) | |
dee5ea7a KS |
38 | extern "C" short _InterlockedCompareExchange16( // NOLINT |
39 | short volatile *Destination, // NOLINT | |
40 | short Exchange, short Comparand); // NOLINT | |
41 | #pragma intrinsic(_InterlockedCompareExchange16) | |
42 | extern "C" | |
43 | long long _InterlockedCompareExchange64( // NOLINT | |
44 | long long volatile *Destination, // NOLINT | |
45 | long long Exchange, long long Comparand); // NOLINT | |
46 | #pragma intrinsic(_InterlockedCompareExchange64) | |
e9772e16 | 47 | extern "C" void *_InterlockedCompareExchangePointer( |
f35db108 WM |
48 | void *volatile *Destination, |
49 | void *Exchange, void *Comparand); | |
e9772e16 | 50 | #pragma intrinsic(_InterlockedCompareExchangePointer) |
e9772e16 KS |
51 | extern "C" |
52 | long __cdecl _InterlockedCompareExchange( // NOLINT | |
53 | long volatile *Destination, // NOLINT | |
54 | long Exchange, long Comparand); // NOLINT | |
55 | #pragma intrinsic(_InterlockedCompareExchange) | |
56 | ||
866e32ad KS |
57 | #ifdef _WIN64 |
58 | extern "C" long long _InterlockedExchangeAdd64( // NOLINT | |
59 | long long volatile * Addend, long long Value); // NOLINT | |
60 | #pragma intrinsic(_InterlockedExchangeAdd64) | |
e9772e16 | 61 | #endif |
f35db108 WM |
62 | |
63 | namespace __sanitizer { | |
64 | ||
65 | INLINE void atomic_signal_fence(memory_order) { | |
66 | _ReadWriteBarrier(); | |
67 | } | |
68 | ||
69 | INLINE void atomic_thread_fence(memory_order) { | |
70 | _mm_mfence(); | |
71 | } | |
72 | ||
73 | INLINE void proc_yield(int cnt) { | |
74 | for (int i = 0; i < cnt; i++) | |
75 | _mm_pause(); | |
76 | } | |
77 | ||
78 | template<typename T> | |
79 | INLINE typename T::Type atomic_load( | |
80 | const volatile T *a, memory_order mo) { | |
81 | DCHECK(mo & (memory_order_relaxed | memory_order_consume | |
82 | | memory_order_acquire | memory_order_seq_cst)); | |
83 | DCHECK(!((uptr)a % sizeof(*a))); | |
84 | typename T::Type v; | |
2660d12d | 85 | // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. |
f35db108 WM |
86 | if (mo == memory_order_relaxed) { |
87 | v = a->val_dont_use; | |
88 | } else { | |
89 | atomic_signal_fence(memory_order_seq_cst); | |
90 | v = a->val_dont_use; | |
91 | atomic_signal_fence(memory_order_seq_cst); | |
92 | } | |
93 | return v; | |
94 | } | |
95 | ||
96 | template<typename T> | |
97 | INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { | |
98 | DCHECK(mo & (memory_order_relaxed | memory_order_release | |
99 | | memory_order_seq_cst)); | |
100 | DCHECK(!((uptr)a % sizeof(*a))); | |
2660d12d | 101 | // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. |
f35db108 WM |
102 | if (mo == memory_order_relaxed) { |
103 | a->val_dont_use = v; | |
104 | } else { | |
105 | atomic_signal_fence(memory_order_seq_cst); | |
106 | a->val_dont_use = v; | |
107 | atomic_signal_fence(memory_order_seq_cst); | |
108 | } | |
109 | if (mo == memory_order_seq_cst) | |
110 | atomic_thread_fence(memory_order_seq_cst); | |
111 | } | |
112 | ||
113 | INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, | |
114 | u32 v, memory_order mo) { | |
115 | (void)mo; | |
116 | DCHECK(!((uptr)a % sizeof(*a))); | |
117 | return (u32)_InterlockedExchangeAdd( | |
118 | (volatile long*)&a->val_dont_use, (long)v); // NOLINT | |
119 | } | |
120 | ||
dee5ea7a KS |
121 | INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, |
122 | uptr v, memory_order mo) { | |
123 | (void)mo; | |
124 | DCHECK(!((uptr)a % sizeof(*a))); | |
125 | #ifdef _WIN64 | |
126 | return (uptr)_InterlockedExchangeAdd64( | |
127 | (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT | |
128 | #else | |
129 | return (uptr)_InterlockedExchangeAdd( | |
130 | (volatile long*)&a->val_dont_use, (long)v); // NOLINT | |
131 | #endif | |
132 | } | |
133 | ||
134 | INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, | |
135 | u32 v, memory_order mo) { | |
136 | (void)mo; | |
137 | DCHECK(!((uptr)a % sizeof(*a))); | |
138 | return (u32)_InterlockedExchangeAdd( | |
139 | (volatile long*)&a->val_dont_use, -(long)v); // NOLINT | |
140 | } | |
141 | ||
142 | INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, | |
143 | uptr v, memory_order mo) { | |
144 | (void)mo; | |
145 | DCHECK(!((uptr)a % sizeof(*a))); | |
146 | #ifdef _WIN64 | |
147 | return (uptr)_InterlockedExchangeAdd64( | |
148 | (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT | |
149 | #else | |
150 | return (uptr)_InterlockedExchangeAdd( | |
151 | (volatile long*)&a->val_dont_use, -(long)v); // NOLINT | |
152 | #endif | |
153 | } | |
154 | ||
f35db108 WM |
155 | INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, |
156 | u8 v, memory_order mo) { | |
157 | (void)mo; | |
158 | DCHECK(!((uptr)a % sizeof(*a))); | |
696d846a | 159 | return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); |
f35db108 WM |
160 | } |
161 | ||
162 | INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, | |
163 | u16 v, memory_order mo) { | |
164 | (void)mo; | |
165 | DCHECK(!((uptr)a % sizeof(*a))); | |
696d846a MO |
166 | return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); |
167 | } | |
168 | ||
169 | INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, | |
170 | u32 v, memory_order mo) { | |
171 | (void)mo; | |
172 | DCHECK(!((uptr)a % sizeof(*a))); | |
173 | return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); | |
f35db108 WM |
174 | } |
175 | ||
ef1b3fda KS |
176 | INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, |
177 | u8 *cmp, | |
178 | u8 xchgv, | |
179 | memory_order mo) { | |
180 | (void)mo; | |
181 | DCHECK(!((uptr)a % sizeof(*a))); | |
182 | u8 cmpv = *cmp; | |
10189819 MO |
183 | #ifdef _WIN64 |
184 | u8 prev = (u8)_InterlockedCompareExchange8( | |
185 | (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv); | |
186 | #else | |
ef1b3fda KS |
187 | u8 prev; |
188 | __asm { | |
189 | mov al, cmpv | |
190 | mov ecx, a | |
191 | mov dl, xchgv | |
192 | lock cmpxchg [ecx], dl | |
193 | mov prev, al | |
194 | } | |
10189819 | 195 | #endif |
ef1b3fda KS |
196 | if (prev == cmpv) |
197 | return true; | |
198 | *cmp = prev; | |
199 | return false; | |
200 | } | |
201 | ||
f35db108 WM |
202 | INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, |
203 | uptr *cmp, | |
204 | uptr xchg, | |
205 | memory_order mo) { | |
206 | uptr cmpv = *cmp; | |
e9772e16 | 207 | uptr prev = (uptr)_InterlockedCompareExchangePointer( |
f35db108 WM |
208 | (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); |
209 | if (prev == cmpv) | |
210 | return true; | |
211 | *cmp = prev; | |
212 | return false; | |
213 | } | |
214 | ||
dee5ea7a KS |
215 | INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, |
216 | u16 *cmp, | |
217 | u16 xchg, | |
218 | memory_order mo) { | |
219 | u16 cmpv = *cmp; | |
220 | u16 prev = (u16)_InterlockedCompareExchange16( | |
221 | (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); | |
222 | if (prev == cmpv) | |
223 | return true; | |
224 | *cmp = prev; | |
225 | return false; | |
226 | } | |
227 | ||
228 | INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, | |
229 | u32 *cmp, | |
230 | u32 xchg, | |
231 | memory_order mo) { | |
232 | u32 cmpv = *cmp; | |
233 | u32 prev = (u32)_InterlockedCompareExchange( | |
234 | (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); | |
235 | if (prev == cmpv) | |
236 | return true; | |
237 | *cmp = prev; | |
238 | return false; | |
239 | } | |
240 | ||
241 | INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, | |
242 | u64 *cmp, | |
243 | u64 xchg, | |
244 | memory_order mo) { | |
245 | u64 cmpv = *cmp; | |
246 | u64 prev = (u64)_InterlockedCompareExchange64( | |
247 | (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); | |
248 | if (prev == cmpv) | |
249 | return true; | |
250 | *cmp = prev; | |
251 | return false; | |
252 | } | |
253 | ||
f35db108 WM |
254 | template<typename T> |
255 | INLINE bool atomic_compare_exchange_weak(volatile T *a, | |
ef1b3fda KS |
256 | typename T::Type *cmp, |
257 | typename T::Type xchg, | |
258 | memory_order mo) { | |
f35db108 WM |
259 | return atomic_compare_exchange_strong(a, cmp, xchg, mo); |
260 | } | |
261 | ||
262 | } // namespace __sanitizer | |
263 | ||
264 | #endif // SANITIZER_ATOMIC_CLANG_H |