]>
Commit | Line | Data |
---|---|---|
03385ed3 | 1 | /** |
2 | * The atomic module provides basic support for lock-free | |
3 | * concurrent programming. | |
4 | * | |
5 | * Copyright: Copyright Sean Kelly 2005 - 2016. | |
6 | * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) | |
7 | * Authors: Sean Kelly, Alex Rønne Petersen | |
8 | * Source: $(DRUNTIMESRC core/_atomic.d) | |
9 | */ | |
10 | ||
11 | ||
12 | /* NOTE: This file has been patched from the original DMD distribution to | |
13 | * work with the GDC compiler. | |
14 | */ | |
15 | module core.atomic; | |
16 | ||
17 | version (D_InlineAsm_X86) | |
18 | { | |
19 | version = AsmX86; | |
20 | version = AsmX86_32; | |
21 | enum has64BitCAS = true; | |
22 | enum has128BitCAS = false; | |
23 | } | |
24 | else version (D_InlineAsm_X86_64) | |
25 | { | |
26 | version = AsmX86; | |
27 | version = AsmX86_64; | |
28 | enum has64BitCAS = true; | |
29 | enum has128BitCAS = true; | |
30 | } | |
31 | else version (GNU) | |
32 | { | |
33 | import gcc.config; | |
34 | enum has64BitCAS = GNU_Have_64Bit_Atomics; | |
35 | enum has128BitCAS = GNU_Have_LibAtomic; | |
36 | } | |
37 | else | |
38 | { | |
39 | enum has64BitCAS = false; | |
40 | enum has128BitCAS = false; | |
41 | } | |
42 | ||
43 | private | |
44 | { | |
45 | template HeadUnshared(T) | |
46 | { | |
47 | static if ( is( T U : shared(U*) ) ) | |
48 | alias shared(U)* HeadUnshared; | |
49 | else | |
50 | alias T HeadUnshared; | |
51 | } | |
52 | } | |
53 | ||
54 | ||
55 | version (AsmX86) | |
56 | { | |
57 | // NOTE: Strictly speaking, the x86 supports atomic operations on | |
58 | // unaligned values. However, this is far slower than the | |
59 | // common case, so such behavior should be prohibited. | |
60 | private bool atomicValueIsProperlyAligned(T)( ref T val ) pure nothrow @nogc @trusted | |
61 | { | |
62 | return atomicPtrIsProperlyAligned(&val); | |
63 | } | |
64 | ||
65 | private bool atomicPtrIsProperlyAligned(T)( T* ptr ) pure nothrow @nogc @safe | |
66 | { | |
67 | // NOTE: 32 bit x86 systems support 8 byte CAS, which only requires | |
68 | // 4 byte alignment, so use size_t as the align type here. | |
69 | static if ( T.sizeof > size_t.sizeof ) | |
70 | return cast(size_t)ptr % size_t.sizeof == 0; | |
71 | else | |
72 | return cast(size_t)ptr % T.sizeof == 0; | |
73 | } | |
74 | } | |
75 | ||
76 | ||
77 | version (CoreDdoc) | |
78 | { | |
79 | /** | |
80 | * Performs the binary operation 'op' on val using 'mod' as the modifier. | |
81 | * | |
82 | * Params: | |
83 | * val = The target variable. | |
84 | * mod = The modifier to apply. | |
85 | * | |
86 | * Returns: | |
87 | * The result of the operation. | |
88 | */ | |
89 | HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe | |
90 | if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) | |
91 | { | |
92 | return HeadUnshared!(T).init; | |
93 | } | |
94 | ||
95 | ||
96 | /** | |
97 | * Stores 'writeThis' to the memory referenced by 'here' if the value | |
98 | * referenced by 'here' is equal to 'ifThis'. This operation is both | |
99 | * lock-free and atomic. | |
100 | * | |
101 | * Params: | |
102 | * here = The address of the destination variable. | |
103 | * writeThis = The value to store. | |
104 | * ifThis = The comparison value. | |
105 | * | |
106 | * Returns: | |
107 | * true if the store occurred, false if not. | |
108 | */ | |
109 | bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
110 | if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); | |
111 | ||
112 | /// Ditto | |
113 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe | |
114 | if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ); | |
115 | ||
116 | /// Ditto | |
117 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe | |
118 | if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); | |
119 | ||
120 | /** | |
121 | * Loads 'val' from memory and returns it. The memory barrier specified | |
122 | * by 'ms' is applied to the operation, which is fully sequenced by | |
123 | * default. Valid memory orders are MemoryOrder.raw, MemoryOrder.acq, | |
124 | * and MemoryOrder.seq. | |
125 | * | |
126 | * Params: | |
127 | * val = The target variable. | |
128 | * | |
129 | * Returns: | |
130 | * The value of 'val'. | |
131 | */ | |
132 | HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq,T)( ref const shared T val ) pure nothrow @nogc @safe | |
133 | { | |
134 | return HeadUnshared!(T).init; | |
135 | } | |
136 | ||
137 | ||
138 | /** | |
139 | * Writes 'newval' into 'val'. The memory barrier specified by 'ms' is | |
140 | * applied to the operation, which is fully sequenced by default. | |
141 | * Valid memory orders are MemoryOrder.raw, MemoryOrder.rel, and | |
142 | * MemoryOrder.seq. | |
143 | * | |
144 | * Params: | |
145 | * val = The target variable. | |
146 | * newval = The value to store. | |
147 | */ | |
148 | void atomicStore(MemoryOrder ms = MemoryOrder.seq,T,V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe | |
149 | if ( __traits( compiles, { val = newval; } ) ) | |
150 | { | |
151 | ||
152 | } | |
153 | ||
154 | ||
155 | /** | |
156 | * Specifies the memory ordering semantics of an atomic operation. | |
157 | */ | |
158 | enum MemoryOrder | |
159 | { | |
160 | raw, /// Not sequenced. | |
161 | acq, /// Hoist-load + hoist-store barrier. | |
162 | rel, /// Sink-load + sink-store barrier. | |
163 | seq, /// Fully sequenced (acquire + release). | |
164 | } | |
165 | ||
166 | deprecated("Please use MemoryOrder instead.") | |
167 | alias MemoryOrder msync; | |
168 | ||
169 | /** | |
170 | * Inserts a full load/store memory fence (on platforms that need it). This ensures | |
171 | * that all loads and stores before a call to this function are executed before any | |
172 | * loads and stores after the call. | |
173 | */ | |
174 | void atomicFence() nothrow @nogc; | |
175 | } | |
176 | else version (AsmX86_32) | |
177 | { | |
178 | // Uses specialized asm for fast fetch and add operations | |
179 | private HeadUnshared!(T) atomicFetchAdd(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe | |
180 | if ( T.sizeof <= 4 ) | |
181 | { | |
182 | size_t tmp = mod; | |
183 | asm pure nothrow @nogc @trusted | |
184 | { | |
185 | mov EAX, tmp; | |
186 | mov EDX, val; | |
187 | } | |
188 | static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AL; } | |
189 | else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AX; } | |
190 | else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[EDX], EAX; } | |
191 | ||
192 | asm pure nothrow @nogc @trusted | |
193 | { | |
194 | mov tmp, EAX; | |
195 | } | |
196 | ||
197 | return cast(T)tmp; | |
198 | } | |
199 | ||
200 | private HeadUnshared!(T) atomicFetchSub(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe | |
201 | if ( T.sizeof <= 4) | |
202 | { | |
203 | return atomicFetchAdd(val, -mod); | |
204 | } | |
205 | ||
206 | HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc | |
207 | if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) | |
208 | in | |
209 | { | |
210 | assert(atomicValueIsProperlyAligned(val)); | |
211 | } | |
212 | body | |
213 | { | |
214 | // binary operators | |
215 | // | |
216 | // + - * / % ^^ & | |
217 | // | ^ << >> >>> ~ in | |
218 | // == != < <= > >= | |
219 | static if ( op == "+" || op == "-" || op == "*" || op == "/" || | |
220 | op == "%" || op == "^^" || op == "&" || op == "|" || | |
221 | op == "^" || op == "<<" || op == ">>" || op == ">>>" || | |
222 | op == "~" || // skip "in" | |
223 | op == "==" || op == "!=" || op == "<" || op == "<=" || | |
224 | op == ">" || op == ">=" ) | |
225 | { | |
226 | HeadUnshared!(T) get = atomicLoad!(MemoryOrder.raw)( val ); | |
227 | mixin( "return get " ~ op ~ " mod;" ); | |
228 | } | |
229 | else | |
230 | // assignment operators | |
231 | // | |
232 | // += -= *= /= %= ^^= &= | |
233 | // |= ^= <<= >>= >>>= ~= | |
234 | static if ( op == "+=" && __traits(isIntegral, T) && T.sizeof <= 4 && V1.sizeof <= 4) | |
235 | { | |
236 | return cast(T)(atomicFetchAdd!(T)(val, mod) + mod); | |
237 | } | |
238 | else static if ( op == "-=" && __traits(isIntegral, T) && T.sizeof <= 4 && V1.sizeof <= 4) | |
239 | { | |
240 | return cast(T)(atomicFetchSub!(T)(val, mod) - mod); | |
241 | } | |
242 | else static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || | |
243 | op == "%=" || op == "^^=" || op == "&=" || op == "|=" || | |
244 | op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" | |
245 | { | |
246 | HeadUnshared!(T) get, set; | |
247 | ||
248 | do | |
249 | { | |
250 | get = set = atomicLoad!(MemoryOrder.raw)( val ); | |
251 | mixin( "set " ~ op ~ " mod;" ); | |
252 | } while ( !casByRef( val, get, set ) ); | |
253 | return set; | |
254 | } | |
255 | else | |
256 | { | |
257 | static assert( false, "Operation not supported." ); | |
258 | } | |
259 | } | |
260 | ||
261 | bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted | |
262 | { | |
263 | return cas(&value, ifThis, writeThis); | |
264 | } | |
265 | ||
266 | bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
267 | if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
268 | { | |
269 | return casImpl(here, ifThis, writeThis); | |
270 | } | |
271 | ||
272 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe | |
273 | if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) | |
274 | { | |
275 | return casImpl(here, ifThis, writeThis); | |
276 | } | |
277 | ||
278 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe | |
279 | if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
280 | { | |
281 | return casImpl(here, ifThis, writeThis); | |
282 | } | |
283 | ||
284 | private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
285 | in | |
286 | { | |
287 | assert( atomicPtrIsProperlyAligned( here ) ); | |
288 | } | |
289 | body | |
290 | { | |
291 | static if ( T.sizeof == byte.sizeof ) | |
292 | { | |
293 | ////////////////////////////////////////////////////////////////// | |
294 | // 1 Byte CAS | |
295 | ////////////////////////////////////////////////////////////////// | |
296 | ||
297 | asm pure nothrow @nogc @trusted | |
298 | { | |
299 | mov DL, writeThis; | |
300 | mov AL, ifThis; | |
301 | mov ECX, here; | |
302 | lock; // lock always needed to make this op atomic | |
303 | cmpxchg [ECX], DL; | |
304 | setz AL; | |
305 | } | |
306 | } | |
307 | else static if ( T.sizeof == short.sizeof ) | |
308 | { | |
309 | ////////////////////////////////////////////////////////////////// | |
310 | // 2 Byte CAS | |
311 | ////////////////////////////////////////////////////////////////// | |
312 | ||
313 | asm pure nothrow @nogc @trusted | |
314 | { | |
315 | mov DX, writeThis; | |
316 | mov AX, ifThis; | |
317 | mov ECX, here; | |
318 | lock; // lock always needed to make this op atomic | |
319 | cmpxchg [ECX], DX; | |
320 | setz AL; | |
321 | } | |
322 | } | |
323 | else static if ( T.sizeof == int.sizeof ) | |
324 | { | |
325 | ////////////////////////////////////////////////////////////////// | |
326 | // 4 Byte CAS | |
327 | ////////////////////////////////////////////////////////////////// | |
328 | ||
329 | asm pure nothrow @nogc @trusted | |
330 | { | |
331 | mov EDX, writeThis; | |
332 | mov EAX, ifThis; | |
333 | mov ECX, here; | |
334 | lock; // lock always needed to make this op atomic | |
335 | cmpxchg [ECX], EDX; | |
336 | setz AL; | |
337 | } | |
338 | } | |
339 | else static if ( T.sizeof == long.sizeof && has64BitCAS ) | |
340 | { | |
341 | ||
342 | ////////////////////////////////////////////////////////////////// | |
343 | // 8 Byte CAS on a 32-Bit Processor | |
344 | ////////////////////////////////////////////////////////////////// | |
345 | ||
346 | asm pure nothrow @nogc @trusted | |
347 | { | |
348 | push EDI; | |
349 | push EBX; | |
350 | lea EDI, writeThis; | |
351 | mov EBX, [EDI]; | |
352 | mov ECX, 4[EDI]; | |
353 | lea EDI, ifThis; | |
354 | mov EAX, [EDI]; | |
355 | mov EDX, 4[EDI]; | |
356 | mov EDI, here; | |
357 | lock; // lock always needed to make this op atomic | |
358 | cmpxchg8b [EDI]; | |
359 | setz AL; | |
360 | pop EBX; | |
361 | pop EDI; | |
362 | ||
363 | } | |
364 | ||
365 | } | |
366 | else | |
367 | { | |
368 | static assert( false, "Invalid template type specified." ); | |
369 | } | |
370 | } | |
371 | ||
372 | ||
373 | enum MemoryOrder | |
374 | { | |
375 | raw, | |
376 | acq, | |
377 | rel, | |
378 | seq, | |
379 | } | |
380 | ||
381 | deprecated("Please use MemoryOrder instead.") | |
382 | alias MemoryOrder msync; | |
383 | ||
384 | ||
385 | private | |
386 | { | |
387 | // NOTE: x86 loads implicitly have acquire semantics so a memory | |
388 | // barrier is only necessary on releases. | |
389 | template needsLoadBarrier( MemoryOrder ms ) | |
390 | { | |
391 | enum bool needsLoadBarrier = ms == MemoryOrder.seq; | |
392 | } | |
393 | ||
394 | ||
395 | // NOTE: x86 stores implicitly have release semantics so a memory | |
396 | // barrier is only necessary on acquires. | |
397 | template needsStoreBarrier( MemoryOrder ms ) | |
398 | { | |
399 | enum bool needsStoreBarrier = ms == MemoryOrder.seq; | |
400 | } | |
401 | } | |
402 | ||
403 | ||
404 | HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe | |
405 | if (!__traits(isFloating, T)) | |
406 | { | |
407 | static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" ); | |
408 | static assert( __traits(isPOD, T), "argument to atomicLoad() must be POD" ); | |
409 | ||
410 | static if ( T.sizeof == byte.sizeof ) | |
411 | { | |
412 | ////////////////////////////////////////////////////////////////// | |
413 | // 1 Byte Load | |
414 | ////////////////////////////////////////////////////////////////// | |
415 | ||
416 | static if ( needsLoadBarrier!(ms) ) | |
417 | { | |
418 | asm pure nothrow @nogc @trusted | |
419 | { | |
420 | mov DL, 0; | |
421 | mov AL, 0; | |
422 | mov ECX, val; | |
423 | lock; // lock always needed to make this op atomic | |
424 | cmpxchg [ECX], DL; | |
425 | } | |
426 | } | |
427 | else | |
428 | { | |
429 | asm pure nothrow @nogc @trusted | |
430 | { | |
431 | mov EAX, val; | |
432 | mov AL, [EAX]; | |
433 | } | |
434 | } | |
435 | } | |
436 | else static if ( T.sizeof == short.sizeof ) | |
437 | { | |
438 | ////////////////////////////////////////////////////////////////// | |
439 | // 2 Byte Load | |
440 | ////////////////////////////////////////////////////////////////// | |
441 | ||
442 | static if ( needsLoadBarrier!(ms) ) | |
443 | { | |
444 | asm pure nothrow @nogc @trusted | |
445 | { | |
446 | mov DX, 0; | |
447 | mov AX, 0; | |
448 | mov ECX, val; | |
449 | lock; // lock always needed to make this op atomic | |
450 | cmpxchg [ECX], DX; | |
451 | } | |
452 | } | |
453 | else | |
454 | { | |
455 | asm pure nothrow @nogc @trusted | |
456 | { | |
457 | mov EAX, val; | |
458 | mov AX, [EAX]; | |
459 | } | |
460 | } | |
461 | } | |
462 | else static if ( T.sizeof == int.sizeof ) | |
463 | { | |
464 | ////////////////////////////////////////////////////////////////// | |
465 | // 4 Byte Load | |
466 | ////////////////////////////////////////////////////////////////// | |
467 | ||
468 | static if ( needsLoadBarrier!(ms) ) | |
469 | { | |
470 | asm pure nothrow @nogc @trusted | |
471 | { | |
472 | mov EDX, 0; | |
473 | mov EAX, 0; | |
474 | mov ECX, val; | |
475 | lock; // lock always needed to make this op atomic | |
476 | cmpxchg [ECX], EDX; | |
477 | } | |
478 | } | |
479 | else | |
480 | { | |
481 | asm pure nothrow @nogc @trusted | |
482 | { | |
483 | mov EAX, val; | |
484 | mov EAX, [EAX]; | |
485 | } | |
486 | } | |
487 | } | |
488 | else static if ( T.sizeof == long.sizeof && has64BitCAS ) | |
489 | { | |
490 | ////////////////////////////////////////////////////////////////// | |
491 | // 8 Byte Load on a 32-Bit Processor | |
492 | ////////////////////////////////////////////////////////////////// | |
493 | ||
494 | asm pure nothrow @nogc @trusted | |
495 | { | |
496 | push EDI; | |
497 | push EBX; | |
498 | mov EBX, 0; | |
499 | mov ECX, 0; | |
500 | mov EAX, 0; | |
501 | mov EDX, 0; | |
502 | mov EDI, val; | |
503 | lock; // lock always needed to make this op atomic | |
504 | cmpxchg8b [EDI]; | |
505 | pop EBX; | |
506 | pop EDI; | |
507 | } | |
508 | } | |
509 | else | |
510 | { | |
511 | static assert( false, "Invalid template type specified." ); | |
512 | } | |
513 | } | |
514 | ||
515 | void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe | |
516 | if ( __traits( compiles, { val = newval; } ) ) | |
517 | { | |
518 | static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" ); | |
519 | static assert( __traits(isPOD, T), "argument to atomicStore() must be POD" ); | |
520 | ||
521 | static if ( T.sizeof == byte.sizeof ) | |
522 | { | |
523 | ////////////////////////////////////////////////////////////////// | |
524 | // 1 Byte Store | |
525 | ////////////////////////////////////////////////////////////////// | |
526 | ||
527 | static if ( needsStoreBarrier!(ms) ) | |
528 | { | |
529 | asm pure nothrow @nogc @trusted | |
530 | { | |
531 | mov EAX, val; | |
532 | mov DL, newval; | |
533 | lock; | |
534 | xchg [EAX], DL; | |
535 | } | |
536 | } | |
537 | else | |
538 | { | |
539 | asm pure nothrow @nogc @trusted | |
540 | { | |
541 | mov EAX, val; | |
542 | mov DL, newval; | |
543 | mov [EAX], DL; | |
544 | } | |
545 | } | |
546 | } | |
547 | else static if ( T.sizeof == short.sizeof ) | |
548 | { | |
549 | ////////////////////////////////////////////////////////////////// | |
550 | // 2 Byte Store | |
551 | ////////////////////////////////////////////////////////////////// | |
552 | ||
553 | static if ( needsStoreBarrier!(ms) ) | |
554 | { | |
555 | asm pure nothrow @nogc @trusted | |
556 | { | |
557 | mov EAX, val; | |
558 | mov DX, newval; | |
559 | lock; | |
560 | xchg [EAX], DX; | |
561 | } | |
562 | } | |
563 | else | |
564 | { | |
565 | asm pure nothrow @nogc @trusted | |
566 | { | |
567 | mov EAX, val; | |
568 | mov DX, newval; | |
569 | mov [EAX], DX; | |
570 | } | |
571 | } | |
572 | } | |
573 | else static if ( T.sizeof == int.sizeof ) | |
574 | { | |
575 | ////////////////////////////////////////////////////////////////// | |
576 | // 4 Byte Store | |
577 | ////////////////////////////////////////////////////////////////// | |
578 | ||
579 | static if ( needsStoreBarrier!(ms) ) | |
580 | { | |
581 | asm pure nothrow @nogc @trusted | |
582 | { | |
583 | mov EAX, val; | |
584 | mov EDX, newval; | |
585 | lock; | |
586 | xchg [EAX], EDX; | |
587 | } | |
588 | } | |
589 | else | |
590 | { | |
591 | asm pure nothrow @nogc @trusted | |
592 | { | |
593 | mov EAX, val; | |
594 | mov EDX, newval; | |
595 | mov [EAX], EDX; | |
596 | } | |
597 | } | |
598 | } | |
599 | else static if ( T.sizeof == long.sizeof && has64BitCAS ) | |
600 | { | |
601 | ////////////////////////////////////////////////////////////////// | |
602 | // 8 Byte Store on a 32-Bit Processor | |
603 | ////////////////////////////////////////////////////////////////// | |
604 | ||
605 | asm pure nothrow @nogc @trusted | |
606 | { | |
607 | push EDI; | |
608 | push EBX; | |
609 | lea EDI, newval; | |
610 | mov EBX, [EDI]; | |
611 | mov ECX, 4[EDI]; | |
612 | mov EDI, val; | |
613 | mov EAX, [EDI]; | |
614 | mov EDX, 4[EDI]; | |
615 | L1: lock; // lock always needed to make this op atomic | |
616 | cmpxchg8b [EDI]; | |
617 | jne L1; | |
618 | pop EBX; | |
619 | pop EDI; | |
620 | } | |
621 | } | |
622 | else | |
623 | { | |
624 | static assert( false, "Invalid template type specified." ); | |
625 | } | |
626 | } | |
627 | ||
628 | ||
629 | void atomicFence() nothrow @nogc @safe | |
630 | { | |
631 | import core.cpuid; | |
632 | ||
633 | asm pure nothrow @nogc @trusted | |
634 | { | |
635 | naked; | |
636 | ||
637 | call sse2; | |
638 | test AL, AL; | |
639 | jne Lcpuid; | |
640 | ||
641 | // Fast path: We have SSE2, so just use mfence. | |
642 | mfence; | |
643 | jmp Lend; | |
644 | ||
645 | Lcpuid: | |
646 | ||
647 | // Slow path: We use cpuid to serialize. This is | |
648 | // significantly slower than mfence, but is the | |
649 | // only serialization facility we have available | |
650 | // on older non-SSE2 chips. | |
651 | push EBX; | |
652 | ||
653 | mov EAX, 0; | |
654 | cpuid; | |
655 | ||
656 | pop EBX; | |
657 | ||
658 | Lend: | |
659 | ||
660 | ret; | |
661 | } | |
662 | } | |
663 | } | |
664 | else version (AsmX86_64) | |
665 | { | |
666 | // Uses specialized asm for fast fetch and add operations | |
667 | private HeadUnshared!(T) atomicFetchAdd(T)( ref shared T val, size_t mod ) pure nothrow @nogc @trusted | |
668 | if ( __traits(isIntegral, T) ) | |
669 | in | |
670 | { | |
671 | assert( atomicValueIsProperlyAligned(val)); | |
672 | } | |
673 | body | |
674 | { | |
675 | size_t tmp = mod; | |
676 | asm pure nothrow @nogc @trusted | |
677 | { | |
678 | mov RAX, tmp; | |
679 | mov RDX, val; | |
680 | } | |
681 | static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AL; } | |
682 | else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AX; } | |
683 | else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[RDX], EAX; } | |
684 | else static if (T.sizeof == 8) asm pure nothrow @nogc @trusted { lock; xadd[RDX], RAX; } | |
685 | ||
686 | asm pure nothrow @nogc @trusted | |
687 | { | |
688 | mov tmp, RAX; | |
689 | } | |
690 | ||
691 | return cast(T)tmp; | |
692 | } | |
693 | ||
694 | private HeadUnshared!(T) atomicFetchSub(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe | |
695 | if ( __traits(isIntegral, T) ) | |
696 | { | |
697 | return atomicFetchAdd(val, -mod); | |
698 | } | |
699 | ||
700 | HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc | |
701 | if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) | |
702 | in | |
703 | { | |
704 | assert( atomicValueIsProperlyAligned(val)); | |
705 | } | |
706 | body | |
707 | { | |
708 | // binary operators | |
709 | // | |
710 | // + - * / % ^^ & | |
711 | // | ^ << >> >>> ~ in | |
712 | // == != < <= > >= | |
713 | static if ( op == "+" || op == "-" || op == "*" || op == "/" || | |
714 | op == "%" || op == "^^" || op == "&" || op == "|" || | |
715 | op == "^" || op == "<<" || op == ">>" || op == ">>>" || | |
716 | op == "~" || // skip "in" | |
717 | op == "==" || op == "!=" || op == "<" || op == "<=" || | |
718 | op == ">" || op == ">=" ) | |
719 | { | |
720 | HeadUnshared!(T) get = atomicLoad!(MemoryOrder.raw)( val ); | |
721 | mixin( "return get " ~ op ~ " mod;" ); | |
722 | } | |
723 | else | |
724 | // assignment operators | |
725 | // | |
726 | // += -= *= /= %= ^^= &= | |
727 | // |= ^= <<= >>= >>>= ~= | |
728 | static if ( op == "+=" && __traits(isIntegral, T) && __traits(isIntegral, V1)) | |
729 | { | |
730 | return cast(T)(atomicFetchAdd!(T)(val, mod) + mod); | |
731 | } | |
732 | else static if ( op == "-=" && __traits(isIntegral, T) && __traits(isIntegral, V1)) | |
733 | { | |
734 | return cast(T)(atomicFetchSub!(T)(val, mod) - mod); | |
735 | } | |
736 | else static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || | |
737 | op == "%=" || op == "^^=" || op == "&=" || op == "|=" || | |
738 | op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" | |
739 | { | |
740 | HeadUnshared!(T) get, set; | |
741 | ||
742 | do | |
743 | { | |
744 | get = set = atomicLoad!(MemoryOrder.raw)( val ); | |
745 | mixin( "set " ~ op ~ " mod;" ); | |
746 | } while ( !casByRef( val, get, set ) ); | |
747 | return set; | |
748 | } | |
749 | else | |
750 | { | |
751 | static assert( false, "Operation not supported." ); | |
752 | } | |
753 | } | |
754 | ||
755 | ||
756 | bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted | |
757 | { | |
758 | return cas(&value, ifThis, writeThis); | |
759 | } | |
760 | ||
761 | bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
762 | if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
763 | { | |
764 | return casImpl(here, ifThis, writeThis); | |
765 | } | |
766 | ||
767 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe | |
768 | if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) | |
769 | { | |
770 | return casImpl(here, ifThis, writeThis); | |
771 | } | |
772 | ||
773 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe | |
774 | if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
775 | { | |
776 | return casImpl(here, ifThis, writeThis); | |
777 | } | |
778 | ||
779 | private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
780 | in | |
781 | { | |
782 | assert( atomicPtrIsProperlyAligned( here ) ); | |
783 | } | |
784 | body | |
785 | { | |
786 | static if ( T.sizeof == byte.sizeof ) | |
787 | { | |
788 | ////////////////////////////////////////////////////////////////// | |
789 | // 1 Byte CAS | |
790 | ////////////////////////////////////////////////////////////////// | |
791 | ||
792 | asm pure nothrow @nogc @trusted | |
793 | { | |
794 | mov DL, writeThis; | |
795 | mov AL, ifThis; | |
796 | mov RCX, here; | |
797 | lock; // lock always needed to make this op atomic | |
798 | cmpxchg [RCX], DL; | |
799 | setz AL; | |
800 | } | |
801 | } | |
802 | else static if ( T.sizeof == short.sizeof ) | |
803 | { | |
804 | ////////////////////////////////////////////////////////////////// | |
805 | // 2 Byte CAS | |
806 | ////////////////////////////////////////////////////////////////// | |
807 | ||
808 | asm pure nothrow @nogc @trusted | |
809 | { | |
810 | mov DX, writeThis; | |
811 | mov AX, ifThis; | |
812 | mov RCX, here; | |
813 | lock; // lock always needed to make this op atomic | |
814 | cmpxchg [RCX], DX; | |
815 | setz AL; | |
816 | } | |
817 | } | |
818 | else static if ( T.sizeof == int.sizeof ) | |
819 | { | |
820 | ////////////////////////////////////////////////////////////////// | |
821 | // 4 Byte CAS | |
822 | ////////////////////////////////////////////////////////////////// | |
823 | ||
824 | asm pure nothrow @nogc @trusted | |
825 | { | |
826 | mov EDX, writeThis; | |
827 | mov EAX, ifThis; | |
828 | mov RCX, here; | |
829 | lock; // lock always needed to make this op atomic | |
830 | cmpxchg [RCX], EDX; | |
831 | setz AL; | |
832 | } | |
833 | } | |
834 | else static if ( T.sizeof == long.sizeof ) | |
835 | { | |
836 | ////////////////////////////////////////////////////////////////// | |
837 | // 8 Byte CAS on a 64-Bit Processor | |
838 | ////////////////////////////////////////////////////////////////// | |
839 | ||
840 | asm pure nothrow @nogc @trusted | |
841 | { | |
842 | mov RDX, writeThis; | |
843 | mov RAX, ifThis; | |
844 | mov RCX, here; | |
845 | lock; // lock always needed to make this op atomic | |
846 | cmpxchg [RCX], RDX; | |
847 | setz AL; | |
848 | } | |
849 | } | |
850 | else static if ( T.sizeof == long.sizeof*2 && has128BitCAS) | |
851 | { | |
852 | ////////////////////////////////////////////////////////////////// | |
853 | // 16 Byte CAS on a 64-Bit Processor | |
854 | ////////////////////////////////////////////////////////////////// | |
855 | version (Win64){ | |
856 | //Windows 64 calling convention uses different registers. | |
857 | //DMD appears to reverse the register order. | |
858 | asm pure nothrow @nogc @trusted | |
859 | { | |
860 | push RDI; | |
861 | push RBX; | |
862 | mov R9, writeThis; | |
863 | mov R10, ifThis; | |
864 | mov R11, here; | |
865 | ||
866 | mov RDI, R9; | |
867 | mov RBX, [RDI]; | |
868 | mov RCX, 8[RDI]; | |
869 | ||
870 | mov RDI, R10; | |
871 | mov RAX, [RDI]; | |
872 | mov RDX, 8[RDI]; | |
873 | ||
874 | mov RDI, R11; | |
875 | lock; | |
876 | cmpxchg16b [RDI]; | |
877 | setz AL; | |
878 | pop RBX; | |
879 | pop RDI; | |
880 | } | |
881 | ||
882 | }else{ | |
883 | ||
884 | asm pure nothrow @nogc @trusted | |
885 | { | |
886 | push RDI; | |
887 | push RBX; | |
888 | lea RDI, writeThis; | |
889 | mov RBX, [RDI]; | |
890 | mov RCX, 8[RDI]; | |
891 | lea RDI, ifThis; | |
892 | mov RAX, [RDI]; | |
893 | mov RDX, 8[RDI]; | |
894 | mov RDI, here; | |
895 | lock; // lock always needed to make this op atomic | |
896 | cmpxchg16b [RDI]; | |
897 | setz AL; | |
898 | pop RBX; | |
899 | pop RDI; | |
900 | } | |
901 | } | |
902 | } | |
903 | else | |
904 | { | |
905 | static assert( false, "Invalid template type specified." ); | |
906 | } | |
907 | } | |
908 | ||
909 | ||
910 | enum MemoryOrder | |
911 | { | |
912 | raw, | |
913 | acq, | |
914 | rel, | |
915 | seq, | |
916 | } | |
917 | ||
918 | deprecated("Please use MemoryOrder instead.") | |
919 | alias MemoryOrder msync; | |
920 | ||
921 | ||
922 | private | |
923 | { | |
924 | // NOTE: x86 loads implicitly have acquire semantics so a memory | |
925 | // barrier is only necessary on releases. | |
926 | template needsLoadBarrier( MemoryOrder ms ) | |
927 | { | |
928 | enum bool needsLoadBarrier = ms == MemoryOrder.seq; | |
929 | } | |
930 | ||
931 | ||
932 | // NOTE: x86 stores implicitly have release semantics so a memory | |
933 | // barrier is only necessary on acquires. | |
934 | template needsStoreBarrier( MemoryOrder ms ) | |
935 | { | |
936 | enum bool needsStoreBarrier = ms == MemoryOrder.seq; | |
937 | } | |
938 | } | |
939 | ||
940 | ||
941 | HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe | |
942 | if (!__traits(isFloating, T)) | |
943 | { | |
944 | static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" ); | |
945 | static assert( __traits(isPOD, T), "argument to atomicLoad() must be POD" ); | |
946 | ||
947 | static if ( T.sizeof == byte.sizeof ) | |
948 | { | |
949 | ////////////////////////////////////////////////////////////////// | |
950 | // 1 Byte Load | |
951 | ////////////////////////////////////////////////////////////////// | |
952 | ||
953 | static if ( needsLoadBarrier!(ms) ) | |
954 | { | |
955 | asm pure nothrow @nogc @trusted | |
956 | { | |
957 | mov DL, 0; | |
958 | mov AL, 0; | |
959 | mov RCX, val; | |
960 | lock; // lock always needed to make this op atomic | |
961 | cmpxchg [RCX], DL; | |
962 | } | |
963 | } | |
964 | else | |
965 | { | |
966 | asm pure nothrow @nogc @trusted | |
967 | { | |
968 | mov RAX, val; | |
969 | mov AL, [RAX]; | |
970 | } | |
971 | } | |
972 | } | |
973 | else static if ( T.sizeof == short.sizeof ) | |
974 | { | |
975 | ////////////////////////////////////////////////////////////////// | |
976 | // 2 Byte Load | |
977 | ////////////////////////////////////////////////////////////////// | |
978 | ||
979 | static if ( needsLoadBarrier!(ms) ) | |
980 | { | |
981 | asm pure nothrow @nogc @trusted | |
982 | { | |
983 | mov DX, 0; | |
984 | mov AX, 0; | |
985 | mov RCX, val; | |
986 | lock; // lock always needed to make this op atomic | |
987 | cmpxchg [RCX], DX; | |
988 | } | |
989 | } | |
990 | else | |
991 | { | |
992 | asm pure nothrow @nogc @trusted | |
993 | { | |
994 | mov RAX, val; | |
995 | mov AX, [RAX]; | |
996 | } | |
997 | } | |
998 | } | |
999 | else static if ( T.sizeof == int.sizeof ) | |
1000 | { | |
1001 | ////////////////////////////////////////////////////////////////// | |
1002 | // 4 Byte Load | |
1003 | ////////////////////////////////////////////////////////////////// | |
1004 | ||
1005 | static if ( needsLoadBarrier!(ms) ) | |
1006 | { | |
1007 | asm pure nothrow @nogc @trusted | |
1008 | { | |
1009 | mov EDX, 0; | |
1010 | mov EAX, 0; | |
1011 | mov RCX, val; | |
1012 | lock; // lock always needed to make this op atomic | |
1013 | cmpxchg [RCX], EDX; | |
1014 | } | |
1015 | } | |
1016 | else | |
1017 | { | |
1018 | asm pure nothrow @nogc @trusted | |
1019 | { | |
1020 | mov RAX, val; | |
1021 | mov EAX, [RAX]; | |
1022 | } | |
1023 | } | |
1024 | } | |
1025 | else static if ( T.sizeof == long.sizeof ) | |
1026 | { | |
1027 | ////////////////////////////////////////////////////////////////// | |
1028 | // 8 Byte Load | |
1029 | ////////////////////////////////////////////////////////////////// | |
1030 | ||
1031 | static if ( needsLoadBarrier!(ms) ) | |
1032 | { | |
1033 | asm pure nothrow @nogc @trusted | |
1034 | { | |
1035 | mov RDX, 0; | |
1036 | mov RAX, 0; | |
1037 | mov RCX, val; | |
1038 | lock; // lock always needed to make this op atomic | |
1039 | cmpxchg [RCX], RDX; | |
1040 | } | |
1041 | } | |
1042 | else | |
1043 | { | |
1044 | asm pure nothrow @nogc @trusted | |
1045 | { | |
1046 | mov RAX, val; | |
1047 | mov RAX, [RAX]; | |
1048 | } | |
1049 | } | |
1050 | } | |
1051 | else static if ( T.sizeof == long.sizeof*2 && has128BitCAS ) | |
1052 | { | |
1053 | ////////////////////////////////////////////////////////////////// | |
1054 | // 16 Byte Load on a 64-Bit Processor | |
1055 | ////////////////////////////////////////////////////////////////// | |
1056 | version (Win64){ | |
1057 | size_t[2] retVal; | |
1058 | asm pure nothrow @nogc @trusted | |
1059 | { | |
1060 | push RDI; | |
1061 | push RBX; | |
1062 | mov RDI, val; | |
1063 | mov RBX, 0; | |
1064 | mov RCX, 0; | |
1065 | mov RAX, 0; | |
1066 | mov RDX, 0; | |
1067 | lock; // lock always needed to make this op atomic | |
1068 | cmpxchg16b [RDI]; | |
1069 | lea RDI, retVal; | |
1070 | mov [RDI], RAX; | |
1071 | mov 8[RDI], RDX; | |
1072 | pop RBX; | |
1073 | pop RDI; | |
1074 | } | |
1075 | ||
1076 | static if (is(T:U[], U)) | |
1077 | { | |
1078 | pragma(inline, true) | |
1079 | static typeof(return) toTrusted(size_t[2] retVal) @trusted | |
1080 | { | |
1081 | return *(cast(typeof(return)*) retVal.ptr); | |
1082 | } | |
1083 | ||
1084 | return toTrusted(retVal); | |
1085 | } | |
1086 | else | |
1087 | { | |
1088 | return cast(typeof(return)) retVal; | |
1089 | } | |
1090 | }else{ | |
1091 | asm pure nothrow @nogc @trusted | |
1092 | { | |
1093 | push RDI; | |
1094 | push RBX; | |
1095 | mov RBX, 0; | |
1096 | mov RCX, 0; | |
1097 | mov RAX, 0; | |
1098 | mov RDX, 0; | |
1099 | mov RDI, val; | |
1100 | lock; // lock always needed to make this op atomic | |
1101 | cmpxchg16b [RDI]; | |
1102 | pop RBX; | |
1103 | pop RDI; | |
1104 | } | |
1105 | } | |
1106 | } | |
1107 | else | |
1108 | { | |
1109 | static assert( false, "Invalid template type specified." ); | |
1110 | } | |
1111 | } | |
1112 | ||
1113 | ||
1114 | void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe | |
1115 | if ( __traits( compiles, { val = newval; } ) ) | |
1116 | { | |
1117 | static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" ); | |
1118 | static assert( __traits(isPOD, T), "argument to atomicStore() must be POD" ); | |
1119 | ||
1120 | static if ( T.sizeof == byte.sizeof ) | |
1121 | { | |
1122 | ////////////////////////////////////////////////////////////////// | |
1123 | // 1 Byte Store | |
1124 | ////////////////////////////////////////////////////////////////// | |
1125 | ||
1126 | static if ( needsStoreBarrier!(ms) ) | |
1127 | { | |
1128 | asm pure nothrow @nogc @trusted | |
1129 | { | |
1130 | mov RAX, val; | |
1131 | mov DL, newval; | |
1132 | lock; | |
1133 | xchg [RAX], DL; | |
1134 | } | |
1135 | } | |
1136 | else | |
1137 | { | |
1138 | asm pure nothrow @nogc @trusted | |
1139 | { | |
1140 | mov RAX, val; | |
1141 | mov DL, newval; | |
1142 | mov [RAX], DL; | |
1143 | } | |
1144 | } | |
1145 | } | |
1146 | else static if ( T.sizeof == short.sizeof ) | |
1147 | { | |
1148 | ////////////////////////////////////////////////////////////////// | |
1149 | // 2 Byte Store | |
1150 | ////////////////////////////////////////////////////////////////// | |
1151 | ||
1152 | static if ( needsStoreBarrier!(ms) ) | |
1153 | { | |
1154 | asm pure nothrow @nogc @trusted | |
1155 | { | |
1156 | mov RAX, val; | |
1157 | mov DX, newval; | |
1158 | lock; | |
1159 | xchg [RAX], DX; | |
1160 | } | |
1161 | } | |
1162 | else | |
1163 | { | |
1164 | asm pure nothrow @nogc @trusted | |
1165 | { | |
1166 | mov RAX, val; | |
1167 | mov DX, newval; | |
1168 | mov [RAX], DX; | |
1169 | } | |
1170 | } | |
1171 | } | |
1172 | else static if ( T.sizeof == int.sizeof ) | |
1173 | { | |
1174 | ////////////////////////////////////////////////////////////////// | |
1175 | // 4 Byte Store | |
1176 | ////////////////////////////////////////////////////////////////// | |
1177 | ||
1178 | static if ( needsStoreBarrier!(ms) ) | |
1179 | { | |
1180 | asm pure nothrow @nogc @trusted | |
1181 | { | |
1182 | mov RAX, val; | |
1183 | mov EDX, newval; | |
1184 | lock; | |
1185 | xchg [RAX], EDX; | |
1186 | } | |
1187 | } | |
1188 | else | |
1189 | { | |
1190 | asm pure nothrow @nogc @trusted | |
1191 | { | |
1192 | mov RAX, val; | |
1193 | mov EDX, newval; | |
1194 | mov [RAX], EDX; | |
1195 | } | |
1196 | } | |
1197 | } | |
1198 | else static if ( T.sizeof == long.sizeof && has64BitCAS ) | |
1199 | { | |
1200 | ////////////////////////////////////////////////////////////////// | |
1201 | // 8 Byte Store on a 64-Bit Processor | |
1202 | ////////////////////////////////////////////////////////////////// | |
1203 | ||
1204 | static if ( needsStoreBarrier!(ms) ) | |
1205 | { | |
1206 | asm pure nothrow @nogc @trusted | |
1207 | { | |
1208 | mov RAX, val; | |
1209 | mov RDX, newval; | |
1210 | lock; | |
1211 | xchg [RAX], RDX; | |
1212 | } | |
1213 | } | |
1214 | else | |
1215 | { | |
1216 | asm pure nothrow @nogc @trusted | |
1217 | { | |
1218 | mov RAX, val; | |
1219 | mov RDX, newval; | |
1220 | mov [RAX], RDX; | |
1221 | } | |
1222 | } | |
1223 | } | |
1224 | else static if ( T.sizeof == long.sizeof*2 && has128BitCAS ) | |
1225 | { | |
1226 | ////////////////////////////////////////////////////////////////// | |
1227 | // 16 Byte Store on a 64-Bit Processor | |
1228 | ////////////////////////////////////////////////////////////////// | |
1229 | version (Win64){ | |
1230 | asm pure nothrow @nogc @trusted | |
1231 | { | |
1232 | push RDI; | |
1233 | push RBX; | |
1234 | mov R9, val; | |
1235 | mov R10, newval; | |
1236 | ||
1237 | mov RDI, R10; | |
1238 | mov RBX, [RDI]; | |
1239 | mov RCX, 8[RDI]; | |
1240 | ||
1241 | mov RDI, R9; | |
1242 | mov RAX, [RDI]; | |
1243 | mov RDX, 8[RDI]; | |
1244 | ||
1245 | L1: lock; // lock always needed to make this op atomic | |
1246 | cmpxchg16b [RDI]; | |
1247 | jne L1; | |
1248 | pop RBX; | |
1249 | pop RDI; | |
1250 | } | |
1251 | }else{ | |
1252 | asm pure nothrow @nogc @trusted | |
1253 | { | |
1254 | push RDI; | |
1255 | push RBX; | |
1256 | lea RDI, newval; | |
1257 | mov RBX, [RDI]; | |
1258 | mov RCX, 8[RDI]; | |
1259 | mov RDI, val; | |
1260 | mov RAX, [RDI]; | |
1261 | mov RDX, 8[RDI]; | |
1262 | L1: lock; // lock always needed to make this op atomic | |
1263 | cmpxchg16b [RDI]; | |
1264 | jne L1; | |
1265 | pop RBX; | |
1266 | pop RDI; | |
1267 | } | |
1268 | } | |
1269 | } | |
1270 | else | |
1271 | { | |
1272 | static assert( false, "Invalid template type specified." ); | |
1273 | } | |
1274 | } | |
1275 | ||
1276 | ||
1277 | void atomicFence() nothrow @nogc @safe | |
1278 | { | |
1279 | // SSE2 is always present in 64-bit x86 chips. | |
1280 | asm nothrow @nogc @trusted | |
1281 | { | |
1282 | naked; | |
1283 | ||
1284 | mfence; | |
1285 | ret; | |
1286 | } | |
1287 | } | |
1288 | } | |
1289 | else version (GNU) | |
1290 | { | |
1291 | import gcc.builtins; | |
1292 | ||
1293 | HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @trusted | |
1294 | if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) | |
1295 | { | |
1296 | // binary operators | |
1297 | // | |
1298 | // + - * / % ^^ & | |
1299 | // | ^ << >> >>> ~ in | |
1300 | // == != < <= > >= | |
1301 | static if ( op == "+" || op == "-" || op == "*" || op == "/" || | |
1302 | op == "%" || op == "^^" || op == "&" || op == "|" || | |
1303 | op == "^" || op == "<<" || op == ">>" || op == ">>>" || | |
1304 | op == "~" || // skip "in" | |
1305 | op == "==" || op == "!=" || op == "<" || op == "<=" || | |
1306 | op == ">" || op == ">=" ) | |
1307 | { | |
1308 | HeadUnshared!(T) get = atomicLoad!(MemoryOrder.raw)( val ); | |
1309 | mixin( "return get " ~ op ~ " mod;" ); | |
1310 | } | |
1311 | else | |
1312 | // assignment operators | |
1313 | // | |
1314 | // += -= *= /= %= ^^= &= | |
1315 | // |= ^= <<= >>= >>>= ~= | |
1316 | static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || | |
1317 | op == "%=" || op == "^^=" || op == "&=" || op == "|=" || | |
1318 | op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" | |
1319 | { | |
1320 | HeadUnshared!(T) get, set; | |
1321 | ||
1322 | do | |
1323 | { | |
1324 | get = set = atomicLoad!(MemoryOrder.raw)( val ); | |
1325 | mixin( "set " ~ op ~ " mod;" ); | |
1326 | } while ( !cas( &val, get, set ) ); | |
1327 | return set; | |
1328 | } | |
1329 | else | |
1330 | { | |
1331 | static assert( false, "Operation not supported." ); | |
1332 | } | |
1333 | } | |
1334 | ||
1335 | ||
1336 | bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe | |
1337 | if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
1338 | { | |
1339 | return casImpl(here, ifThis, writeThis); | |
1340 | } | |
1341 | ||
1342 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe | |
1343 | if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) | |
1344 | { | |
1345 | return casImpl(here, ifThis, writeThis); | |
1346 | } | |
1347 | ||
1348 | bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe | |
1349 | if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) | |
1350 | { | |
1351 | return casImpl(here, ifThis, writeThis); | |
1352 | } | |
1353 | ||
1354 | private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted | |
1355 | { | |
03385ed3 | 1356 | bool res = void; |
1357 | ||
cad9f085 | 1358 | static if (GNU_Have_Atomics || GNU_Have_LibAtomic) |
03385ed3 | 1359 | { |
cad9f085 | 1360 | static if (T.sizeof == byte.sizeof) |
1361 | { | |
1362 | res = __atomic_compare_exchange_1(here, cast(void*) &ifThis, *cast(ubyte*) &writeThis, | |
1363 | false, MemoryOrder.seq, MemoryOrder.seq); | |
1364 | } | |
1365 | else static if (T.sizeof == short.sizeof) | |
1366 | { | |
1367 | res = __atomic_compare_exchange_2(here, cast(void*) &ifThis, *cast(ushort*) &writeThis, | |
1368 | false, MemoryOrder.seq, MemoryOrder.seq); | |
1369 | } | |
1370 | else static if (T.sizeof == int.sizeof) | |
1371 | { | |
1372 | res = __atomic_compare_exchange_4(here, cast(void*) &ifThis, *cast(uint*) &writeThis, | |
1373 | false, MemoryOrder.seq, MemoryOrder.seq); | |
1374 | } | |
1375 | else static if (T.sizeof == long.sizeof && GNU_Have_64Bit_Atomics) | |
1376 | { | |
1377 | res = __atomic_compare_exchange_8(here, cast(void*) &ifThis, *cast(ulong*) &writeThis, | |
1378 | false, MemoryOrder.seq, MemoryOrder.seq); | |
1379 | } | |
1380 | else static if (GNU_Have_LibAtomic) | |
1381 | { | |
1382 | res = __atomic_compare_exchange(T.sizeof, here, cast(void*) &ifThis, cast(void*) &writeThis, | |
1383 | MemoryOrder.seq, MemoryOrder.seq); | |
1384 | } | |
1385 | else | |
1386 | static assert(0, "Invalid template type specified."); | |
03385ed3 | 1387 | } |
cad9f085 | 1388 | else |
03385ed3 | 1389 | { |
cad9f085 | 1390 | static if (T.sizeof == byte.sizeof) |
1391 | alias U = byte; | |
1392 | else static if (T.sizeof == short.sizeof) | |
1393 | alias U = short; | |
1394 | else static if (T.sizeof == int.sizeof) | |
1395 | alias U = int; | |
1396 | else static if (T.sizeof == long.sizeof) | |
1397 | alias U = long; | |
1398 | else | |
1399 | static assert(0, "Invalid template type specified."); | |
1400 | ||
1401 | getAtomicMutex.lock(); | |
1402 | scope(exit) getAtomicMutex.unlock(); | |
1403 | ||
1404 | if (*cast(U*)here == *cast(U*)&ifThis) | |
1405 | { | |
1406 | *here = writeThis; | |
1407 | res = true; | |
1408 | } | |
1409 | else | |
1410 | res = false; | |
03385ed3 | 1411 | } |
03385ed3 | 1412 | |
1413 | return res; | |
1414 | } | |
1415 | ||
1416 | ||
1417 | // Memory model types for the __atomic* builtins. | |
1418 | enum MemoryOrder | |
1419 | { | |
1420 | raw = 0, | |
1421 | acq = 2, | |
1422 | rel = 3, | |
1423 | seq = 5, | |
1424 | } | |
1425 | ||
1426 | deprecated("Please use MemoryOrder instead.") | |
1427 | alias MemoryOrder msync; | |
1428 | ||
1429 | ||
1430 | HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @trusted | |
1431 | if (!__traits(isFloating, T)) | |
1432 | { | |
1433 | static assert(ms != MemoryOrder.rel, "Invalid MemoryOrder for atomicLoad"); | |
1434 | static assert(__traits(isPOD, T), "argument to atomicLoad() must be POD"); | |
03385ed3 | 1435 | |
cad9f085 | 1436 | static if (GNU_Have_Atomics || GNU_Have_LibAtomic) |
03385ed3 | 1437 | { |
cad9f085 | 1438 | static if (T.sizeof == ubyte.sizeof) |
1439 | { | |
1440 | ubyte value = __atomic_load_1(&val, ms); | |
1441 | return *cast(HeadUnshared!T*) &value; | |
1442 | } | |
1443 | else static if (T.sizeof == ushort.sizeof) | |
1444 | { | |
1445 | ushort value = __atomic_load_2(&val, ms); | |
1446 | return *cast(HeadUnshared!T*) &value; | |
1447 | } | |
1448 | else static if (T.sizeof == uint.sizeof) | |
1449 | { | |
1450 | uint value = __atomic_load_4(&val, ms); | |
1451 | return *cast(HeadUnshared!T*) &value; | |
1452 | } | |
1453 | else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics) | |
1454 | { | |
1455 | ulong value = __atomic_load_8(&val, ms); | |
1456 | return *cast(HeadUnshared!T*) &value; | |
1457 | } | |
1458 | else static if (GNU_Have_LibAtomic) | |
1459 | { | |
1460 | T value; | |
1461 | __atomic_load(T.sizeof, &val, cast(void*)&value, ms); | |
1462 | return *cast(HeadUnshared!T*) &value; | |
1463 | } | |
1464 | else | |
1465 | static assert(0, "Invalid template type specified."); | |
03385ed3 | 1466 | } |
cad9f085 | 1467 | else |
03385ed3 | 1468 | { |
cad9f085 | 1469 | getAtomicMutex.lock(); |
1470 | scope(exit) getAtomicMutex.unlock(); | |
1471 | return *cast(HeadUnshared!T*)&val; | |
03385ed3 | 1472 | } |
03385ed3 | 1473 | } |
1474 | ||
1475 | ||
1476 | void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @trusted | |
1477 | if ( __traits( compiles, { val = newval; } ) ) | |
1478 | { | |
1479 | static assert(ms != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore"); | |
1480 | static assert(__traits(isPOD, T), "argument to atomicLoad() must be POD"); | |
03385ed3 | 1481 | |
cad9f085 | 1482 | static if (GNU_Have_Atomics || GNU_Have_LibAtomic) |
03385ed3 | 1483 | { |
cad9f085 | 1484 | static if (T.sizeof == ubyte.sizeof) |
1485 | { | |
1486 | __atomic_store_1(&val, *cast(ubyte*) &newval, ms); | |
1487 | } | |
1488 | else static if (T.sizeof == ushort.sizeof) | |
1489 | { | |
1490 | __atomic_store_2(&val, *cast(ushort*) &newval, ms); | |
1491 | } | |
1492 | else static if (T.sizeof == uint.sizeof) | |
1493 | { | |
1494 | __atomic_store_4(&val, *cast(uint*) &newval, ms); | |
1495 | } | |
1496 | else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics) | |
1497 | { | |
1498 | __atomic_store_8(&val, *cast(ulong*) &newval, ms); | |
1499 | } | |
1500 | else static if (GNU_Have_LibAtomic) | |
1501 | { | |
1502 | __atomic_store(T.sizeof, &val, cast(void*)&newval, ms); | |
1503 | } | |
1504 | else | |
1505 | static assert(0, "Invalid template type specified."); | |
03385ed3 | 1506 | } |
cad9f085 | 1507 | else |
03385ed3 | 1508 | { |
cad9f085 | 1509 | getAtomicMutex.lock(); |
1510 | val = newval; | |
1511 | getAtomicMutex.unlock(); | |
03385ed3 | 1512 | } |
cad9f085 | 1513 | } |
1514 | ||
1515 | ||
1516 | void atomicFence() nothrow @nogc | |
1517 | { | |
1518 | static if (GNU_Have_Atomics || GNU_Have_LibAtomic) | |
1519 | __atomic_thread_fence(MemoryOrder.seq); | |
1520 | else | |
03385ed3 | 1521 | { |
cad9f085 | 1522 | getAtomicMutex.lock(); |
1523 | getAtomicMutex.unlock(); | |
03385ed3 | 1524 | } |
cad9f085 | 1525 | } |
1526 | ||
1527 | static if (!GNU_Have_Atomics && !GNU_Have_LibAtomic) | |
1528 | { | |
1529 | // Use system mutex for atomics, faking the purity of the functions so | |
1530 | // that they can be used in pure/nothrow/@safe code. | |
1531 | extern (C) private pure @trusted @nogc nothrow | |
03385ed3 | 1532 | { |
cad9f085 | 1533 | static if (GNU_Thread_Model == ThreadModel.Posix) |
1534 | { | |
1535 | import core.sys.posix.pthread; | |
1536 | alias atomicMutexHandle = pthread_mutex_t; | |
1537 | ||
1538 | pragma(mangle, "pthread_mutex_init") int fakePureMutexInit(pthread_mutex_t*, pthread_mutexattr_t*); | |
1539 | pragma(mangle, "pthread_mutex_lock") int fakePureMutexLock(pthread_mutex_t*); | |
1540 | pragma(mangle, "pthread_mutex_unlock") int fakePureMutexUnlock(pthread_mutex_t*); | |
1541 | } | |
1542 | else static if (GNU_Thread_Model == ThreadModel.Win32) | |
1543 | { | |
1544 | import core.sys.windows.winbase; | |
1545 | alias atomicMutexHandle = CRITICAL_SECTION; | |
1546 | ||
1547 | pragma(mangle, "InitializeCriticalSection") int fakePureMutexInit(CRITICAL_SECTION*); | |
1548 | pragma(mangle, "EnterCriticalSection") void fakePureMutexLock(CRITICAL_SECTION*); | |
1549 | pragma(mangle, "LeaveCriticalSection") int fakePureMutexUnlock(CRITICAL_SECTION*); | |
1550 | } | |
1551 | else | |
1552 | { | |
1553 | alias atomicMutexHandle = int; | |
1554 | } | |
03385ed3 | 1555 | } |
cad9f085 | 1556 | |
1557 | // Implements lock/unlock operations. | |
1558 | private struct AtomicMutex | |
03385ed3 | 1559 | { |
cad9f085 | 1560 | int lock() pure @trusted @nogc nothrow |
1561 | { | |
1562 | static if (GNU_Thread_Model == ThreadModel.Posix) | |
1563 | { | |
1564 | if (!_inited) | |
1565 | { | |
1566 | fakePureMutexInit(&_handle, null); | |
1567 | _inited = true; | |
1568 | } | |
1569 | return fakePureMutexLock(&_handle); | |
1570 | } | |
1571 | else | |
1572 | { | |
1573 | static if (GNU_Thread_Model == ThreadModel.Win32) | |
1574 | { | |
1575 | if (!_inited) | |
1576 | { | |
1577 | fakePureMutexInit(&_handle); | |
1578 | _inited = true; | |
1579 | } | |
1580 | fakePureMutexLock(&_handle); | |
1581 | } | |
1582 | return 0; | |
1583 | } | |
1584 | } | |
1585 | ||
1586 | int unlock() pure @trusted @nogc nothrow | |
1587 | { | |
1588 | static if (GNU_Thread_Model == ThreadModel.Posix) | |
1589 | return fakePureMutexUnlock(&_handle); | |
1590 | else | |
1591 | { | |
1592 | static if (GNU_Thread_Model == ThreadModel.Win32) | |
1593 | fakePureMutexUnlock(&_handle); | |
1594 | return 0; | |
1595 | } | |
1596 | } | |
1597 | ||
1598 | private: | |
1599 | atomicMutexHandle _handle; | |
1600 | bool _inited; | |
03385ed3 | 1601 | } |
03385ed3 | 1602 | |
cad9f085 | 1603 | // Internal static mutex reference. |
1604 | private AtomicMutex* _getAtomicMutex() @trusted @nogc nothrow | |
1605 | { | |
1606 | __gshared static AtomicMutex mutex; | |
1607 | return &mutex; | |
1608 | } | |
03385ed3 | 1609 | |
cad9f085 | 1610 | // Pure alias for _getAtomicMutex. |
1611 | pragma(mangle, _getAtomicMutex.mangleof) | |
1612 | private AtomicMutex* getAtomicMutex() pure @trusted @nogc nothrow @property; | |
03385ed3 | 1613 | } |
1614 | } | |
1615 | ||
1616 | // This is an ABI adapter that works on all architectures. It type puns | |
1617 | // floats and doubles to ints and longs, atomically loads them, then puns | |
1618 | // them back. This is necessary so that they get returned in floating | |
1619 | // point instead of integer registers. | |
1620 | HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @trusted | |
1621 | if (__traits(isFloating, T)) | |
1622 | { | |
1623 | static if (T.sizeof == int.sizeof) | |
1624 | { | |
1625 | static assert(is(T : float)); | |
1626 | auto ptr = cast(const shared int*) &val; | |
1627 | auto asInt = atomicLoad!(ms)(*ptr); | |
1628 | return *(cast(typeof(return)*) &asInt); | |
1629 | } | |
1630 | else static if (T.sizeof == long.sizeof) | |
1631 | { | |
1632 | static assert(is(T : double)); | |
1633 | auto ptr = cast(const shared long*) &val; | |
1634 | auto asLong = atomicLoad!(ms)(*ptr); | |
1635 | return *(cast(typeof(return)*) &asLong); | |
1636 | } | |
1637 | else | |
1638 | { | |
1639 | static assert(0, "Cannot atomically load 80-bit reals."); | |
1640 | } | |
1641 | } | |
1642 | ||
1643 | //////////////////////////////////////////////////////////////////////////////// | |
1644 | // Unit Tests | |
1645 | //////////////////////////////////////////////////////////////////////////////// | |
1646 | ||
1647 | ||
1648 | version (unittest) | |
1649 | { | |
1650 | void testCAS(T)( T val ) pure nothrow @nogc @trusted | |
1651 | in | |
1652 | { | |
1653 | assert(val !is T.init); | |
1654 | } | |
1655 | body | |
1656 | { | |
1657 | T base = cast(T)null; | |
1658 | shared(T) atom = cast(shared(T))null; | |
1659 | ||
1660 | assert( base !is val, T.stringof ); | |
1661 | assert( atom is base, T.stringof ); | |
1662 | ||
1663 | assert( cas( &atom, base, val ), T.stringof ); | |
1664 | assert( atom is val, T.stringof ); | |
1665 | assert( !cas( &atom, base, base ), T.stringof ); | |
1666 | assert( atom is val, T.stringof ); | |
1667 | } | |
1668 | ||
1669 | void testLoadStore(MemoryOrder ms = MemoryOrder.seq, T)( T val = T.init + 1 ) pure nothrow @nogc @trusted | |
1670 | { | |
1671 | T base = cast(T) 0; | |
1672 | shared(T) atom = cast(T) 0; | |
1673 | ||
1674 | assert( base !is val ); | |
1675 | assert( atom is base ); | |
1676 | atomicStore!(ms)( atom, val ); | |
1677 | base = atomicLoad!(ms)( atom ); | |
1678 | ||
1679 | assert( base is val, T.stringof ); | |
1680 | assert( atom is val ); | |
1681 | } | |
1682 | ||
1683 | ||
1684 | void testType(T)( T val = T.init + 1 ) pure nothrow @nogc @safe | |
1685 | { | |
1686 | testCAS!(T)( val ); | |
1687 | testLoadStore!(MemoryOrder.seq, T)( val ); | |
1688 | testLoadStore!(MemoryOrder.raw, T)( val ); | |
1689 | } | |
1690 | ||
1691 | @safe pure nothrow unittest | |
1692 | { | |
1693 | testType!(bool)(); | |
1694 | ||
1695 | testType!(byte)(); | |
1696 | testType!(ubyte)(); | |
1697 | ||
1698 | testType!(short)(); | |
1699 | testType!(ushort)(); | |
1700 | ||
1701 | testType!(int)(); | |
1702 | testType!(uint)(); | |
1703 | ||
1704 | testType!(shared int*)(); | |
1705 | ||
1706 | static class Klass {} | |
1707 | testCAS!(shared Klass)( new shared(Klass) ); | |
1708 | ||
1709 | testType!(float)(1.0f); | |
1710 | ||
1711 | static if ( has64BitCAS ) | |
1712 | { | |
1713 | testType!(double)(1.0); | |
1714 | testType!(long)(); | |
1715 | testType!(ulong)(); | |
1716 | } | |
1717 | ||
1718 | shared(size_t) i; | |
1719 | ||
1720 | atomicOp!"+="( i, cast(size_t) 1 ); | |
1721 | assert( i == 1 ); | |
1722 | ||
1723 | atomicOp!"-="( i, cast(size_t) 1 ); | |
1724 | assert( i == 0 ); | |
1725 | ||
1726 | shared float f = 0; | |
1727 | atomicOp!"+="( f, 1 ); | |
1728 | assert( f == 1 ); | |
1729 | ||
1730 | static if ( has64BitCAS ) | |
1731 | { | |
1732 | shared double d = 0; | |
1733 | atomicOp!"+="( d, 1 ); | |
1734 | assert( d == 1 ); | |
1735 | } | |
1736 | } | |
1737 | ||
1738 | pure nothrow unittest | |
1739 | { | |
1740 | static if (has128BitCAS) | |
1741 | { | |
1742 | struct DoubleValue | |
1743 | { | |
1744 | long value1; | |
1745 | long value2; | |
1746 | } | |
1747 | ||
1748 | align(16) shared DoubleValue a; | |
1749 | atomicStore(a, DoubleValue(1,2)); | |
1750 | assert(a.value1 == 1 && a.value2 ==2); | |
1751 | ||
1752 | while (!cas(&a, DoubleValue(1,2), DoubleValue(3,4))){} | |
1753 | assert(a.value1 == 3 && a.value2 ==4); | |
1754 | ||
1755 | align(16) DoubleValue b = atomicLoad(a); | |
1756 | assert(b.value1 == 3 && b.value2 ==4); | |
1757 | } | |
1758 | ||
1759 | version (D_LP64) | |
1760 | { | |
1761 | enum hasDWCAS = has128BitCAS; | |
1762 | } | |
1763 | else | |
1764 | { | |
1765 | enum hasDWCAS = has64BitCAS; | |
1766 | } | |
1767 | ||
1768 | static if (hasDWCAS) | |
1769 | { | |
1770 | static struct List { size_t gen; List* next; } | |
1771 | shared(List) head; | |
1772 | assert(cas(&head, shared(List)(0, null), shared(List)(1, cast(List*)1))); | |
1773 | assert(head.gen == 1); | |
1774 | assert(cast(size_t)head.next == 1); | |
1775 | } | |
1776 | } | |
1777 | ||
1778 | pure nothrow unittest | |
1779 | { | |
1780 | static struct S { int val; } | |
1781 | auto s = shared(S)(1); | |
1782 | ||
1783 | shared(S*) ptr; | |
1784 | ||
1785 | // head unshared | |
1786 | shared(S)* ifThis = null; | |
1787 | shared(S)* writeThis = &s; | |
1788 | assert(ptr is null); | |
1789 | assert(cas(&ptr, ifThis, writeThis)); | |
1790 | assert(ptr is writeThis); | |
1791 | ||
1792 | // head shared | |
1793 | shared(S*) ifThis2 = writeThis; | |
1794 | shared(S*) writeThis2 = null; | |
1795 | assert(cas(&ptr, ifThis2, writeThis2)); | |
1796 | assert(ptr is null); | |
1797 | ||
1798 | // head unshared target doesn't want atomic CAS | |
1799 | shared(S)* ptr2; | |
1800 | static assert(!__traits(compiles, cas(&ptr2, ifThis, writeThis))); | |
1801 | static assert(!__traits(compiles, cas(&ptr2, ifThis2, writeThis2))); | |
1802 | } | |
1803 | ||
1804 | unittest | |
1805 | { | |
1806 | import core.thread; | |
1807 | ||
1808 | // Use heap memory to ensure an optimizing | |
1809 | // compiler doesn't put things in registers. | |
1810 | uint* x = new uint(); | |
1811 | bool* f = new bool(); | |
1812 | uint* r = new uint(); | |
1813 | ||
1814 | auto thr = new Thread(() | |
1815 | { | |
1816 | while (!*f) | |
1817 | { | |
1818 | } | |
1819 | ||
1820 | atomicFence(); | |
1821 | ||
1822 | *r = *x; | |
1823 | }); | |
1824 | ||
1825 | thr.start(); | |
1826 | ||
1827 | *x = 42; | |
1828 | ||
1829 | atomicFence(); | |
1830 | ||
1831 | *f = true; | |
1832 | ||
1833 | atomicFence(); | |
1834 | ||
1835 | thr.join(); | |
1836 | ||
1837 | assert(*r == 42); | |
1838 | } | |
1839 | ||
1840 | // === atomicFetchAdd and atomicFetchSub operations ==== | |
1841 | pure nothrow @nogc @safe unittest | |
1842 | { | |
1843 | shared ubyte u8 = 1; | |
1844 | shared ushort u16 = 2; | |
1845 | shared uint u32 = 3; | |
1846 | shared byte i8 = 5; | |
1847 | shared short i16 = 6; | |
1848 | shared int i32 = 7; | |
1849 | ||
1850 | assert(atomicOp!"+="(u8, 8) == 9); | |
1851 | assert(atomicOp!"+="(u16, 8) == 10); | |
1852 | assert(atomicOp!"+="(u32, 8) == 11); | |
1853 | assert(atomicOp!"+="(i8, 8) == 13); | |
1854 | assert(atomicOp!"+="(i16, 8) == 14); | |
1855 | assert(atomicOp!"+="(i32, 8) == 15); | |
1856 | version (AsmX86_64) | |
1857 | { | |
1858 | shared ulong u64 = 4; | |
1859 | shared long i64 = 8; | |
1860 | assert(atomicOp!"+="(u64, 8) == 12); | |
1861 | assert(atomicOp!"+="(i64, 8) == 16); | |
1862 | } | |
1863 | } | |
1864 | ||
1865 | pure nothrow @nogc @safe unittest | |
1866 | { | |
1867 | shared ubyte u8 = 1; | |
1868 | shared ushort u16 = 2; | |
1869 | shared uint u32 = 3; | |
1870 | shared byte i8 = 5; | |
1871 | shared short i16 = 6; | |
1872 | shared int i32 = 7; | |
1873 | ||
1874 | assert(atomicOp!"-="(u8, 1) == 0); | |
1875 | assert(atomicOp!"-="(u16, 1) == 1); | |
1876 | assert(atomicOp!"-="(u32, 1) == 2); | |
1877 | assert(atomicOp!"-="(i8, 1) == 4); | |
1878 | assert(atomicOp!"-="(i16, 1) == 5); | |
1879 | assert(atomicOp!"-="(i32, 1) == 6); | |
1880 | version (AsmX86_64) | |
1881 | { | |
1882 | shared ulong u64 = 4; | |
1883 | shared long i64 = 8; | |
1884 | assert(atomicOp!"-="(u64, 1) == 3); | |
1885 | assert(atomicOp!"-="(i64, 1) == 7); | |
1886 | } | |
1887 | } | |
1888 | ||
1889 | pure nothrow @nogc @safe unittest // issue 16651 | |
1890 | { | |
1891 | shared ulong a = 2; | |
1892 | uint b = 1; | |
1893 | atomicOp!"-="( a, b ); | |
1894 | assert(a == 1); | |
1895 | ||
1896 | shared uint c = 2; | |
1897 | ubyte d = 1; | |
1898 | atomicOp!"-="( c, d ); | |
1899 | assert(c == 1); | |
1900 | } | |
1901 | } |