]>
Commit | Line | Data |
---|---|---|
ede3e665 | 1 | /* |
fecb3aae | 2 | * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. |
ede3e665 | 3 | * |
48f4ad77 | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
ede3e665 AP |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
8 | */ | |
df443918 | 9 | |
ede3e665 | 10 | /* |
a88e328c AP |
11 | * Contemporary compilers implement lock-free atomic memory access |
12 | * primitives that facilitate writing "thread-opportunistic" or even real | |
13 | * multi-threading low-overhead code. "Thread-opportunistic" is when | |
ede3e665 AP |
14 | * exact result is not required, e.g. some statistics, or execution flow |
15 | * doesn't have to be unambiguous. Simplest example is lazy "constant" | |
16 | * initialization when one can synchronize on variable itself, e.g. | |
17 | * | |
18 | * if (var == NOT_YET_INITIALIZED) | |
19 | * var = function_returning_same_value(); | |
20 | * | |
c2969ff6 | 21 | * This does work provided that loads and stores are single-instruction |
ede3e665 AP |
22 | * operations (and integer ones are on *all* supported platforms), but |
23 | * it upsets Thread Sanitizer. Suggested solution is | |
24 | * | |
25 | * if (tsan_load(&var) == NOT_YET_INITIALIZED) | |
26 | * tsan_store(&var, function_returning_same_value()); | |
27 | * | |
28 | * Production machine code would be the same, so one can wonder why | |
29 | * bother. Having Thread Sanitizer accept "thread-opportunistic" code | |
30 | * allows to move on trouble-shooting real bugs. | |
31 | * | |
a88e328c AP |
32 | * Resolving Thread Sanitizer nits was the initial purpose for this module, |
33 | * but it was later extended with more nuanced primitives that are useful | |
34 | * even in "non-opportunistic" scenarios. Most notably verifying if a shared | |
35 | * structure is fully initialized and bypassing the initialization lock. | |
36 | * It's suggested to view macros defined in this module as "annotations" for | |
37 | * thread-safe lock-free code, "Thread-Safe ANnotations"... | |
38 | * | |
39 | * It's assumed that ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as | |
40 | * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that corresponding | |
ede3e665 AP |
41 | * code is inlined. It should be noted that statistics counters become |
42 | * accurate in such case. | |
a88e328c AP |
43 | * |
44 | * Special note about TSAN_QUALIFIER. It might be undesired to use it in | |
45 | * a shared header. Because whether operation on specific variable or member | |
46 | * is atomic or not might be irrelevant in other modules. In such case one | |
47 | * can use TSAN_QUALIFIER in cast specifically when it has to count. | |
ede3e665 AP |
48 | */ |
49 | ||
3d27ac8d WL |
50 | #ifndef OSSL_INTERNAL_TSAN_ASSIST_H |
51 | # define OSSL_INTERNAL_TSAN_ASSIST_H | |
52 | # pragma once | |
53 | ||
54 | # if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ | |
ede3e665 | 55 | && !defined(__STDC_NO_ATOMICS__) |
3d27ac8d | 56 | # include <stdatomic.h> |
ede3e665 | 57 | |
3d27ac8d | 58 | # if defined(ATOMIC_POINTER_LOCK_FREE) \ |
ede3e665 | 59 | && ATOMIC_POINTER_LOCK_FREE >= 2 |
3d27ac8d WL |
60 | # define TSAN_QUALIFIER _Atomic |
61 | # define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed) | |
62 | # define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed) | |
63 | # define tsan_add(ptr, n) atomic_fetch_add_explicit((ptr), (n), memory_order_relaxed) | |
64 | # define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire) | |
65 | # define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release) | |
66 | # endif | |
ede3e665 | 67 | |
3d27ac8d | 68 | # elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) |
ede3e665 | 69 | |
3d27ac8d | 70 | # if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \ |
ede3e665 | 71 | && __GCC_ATOMIC_POINTER_LOCK_FREE >= 2 |
3d27ac8d WL |
72 | # define TSAN_QUALIFIER volatile |
73 | # define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED) | |
74 | # define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED) | |
75 | # define tsan_add(ptr, n) __atomic_fetch_add((ptr), (n), __ATOMIC_RELAXED) | |
76 | # define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE) | |
77 | # define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE) | |
78 | # endif | |
ede3e665 | 79 | |
3d27ac8d | 80 | # elif defined(_MSC_VER) && _MSC_VER>=1200 \ |
a88e328c | 81 | && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ |
88ffc8de | 82 | defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7 && !defined(_WIN32_WCE))) |
a88e328c AP |
83 | /* |
84 | * There is subtle dependency on /volatile:<iso|ms> command-line option. | |
85 | * "ms" implies same semantic as memory_order_acquire for loads and | |
86 | * memory_order_release for stores, while "iso" - memory_order_relaxed for | |
87 | * either. Real complication is that defaults are different on x86 and ARM. | |
88 | * There is explanation for that, "ms" is backward compatible with earlier | |
89 | * compiler versions, while multi-processor ARM can be viewed as brand new | |
90 | * platform to MSC and its users, and with non-relaxed semantic taking toll | |
91 | * with additional instructions and penalties, it kind of makes sense to | |
92 | * default to "iso"... | |
93 | */ | |
3d27ac8d WL |
94 | # define TSAN_QUALIFIER volatile |
95 | # if defined(_M_ARM) || defined(_M_ARM64) | |
96 | # define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf | |
97 | # pragma intrinsic(_InterlockedExchangeAdd_nf) | |
98 | # pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32) | |
99 | # ifdef _WIN64 | |
100 | # define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf | |
101 | # pragma intrinsic(_InterlockedExchangeAdd64_nf) | |
102 | # pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64) | |
103 | # define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \ | |
a88e328c | 104 | : __iso_volatile_load32(ptr)) |
3d27ac8d | 105 | # define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \ |
a88e328c | 106 | : __iso_volatile_store32((ptr), (val))) |
3d27ac8d WL |
107 | # else |
108 | # define tsan_load(ptr) __iso_volatile_load32(ptr) | |
109 | # define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val)) | |
110 | # endif | |
a88e328c | 111 | # else |
3d27ac8d WL |
112 | # define tsan_load(ptr) (*(ptr)) |
113 | # define tsan_store(ptr, val) (*(ptr) = (val)) | |
a88e328c | 114 | # endif |
3d27ac8d WL |
115 | # pragma intrinsic(_InterlockedExchangeAdd) |
116 | # ifdef _WIN64 | |
117 | # pragma intrinsic(_InterlockedExchangeAdd64) | |
118 | # define tsan_add(ptr, n) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), (n)) \ | |
b0b456f8 | 119 | : _InterlockedExchangeAdd((ptr), (n))) |
3d27ac8d WL |
120 | # else |
121 | # define tsan_add(ptr, n) _InterlockedExchangeAdd((ptr), (n)) | |
122 | # endif | |
123 | # if !defined(_ISO_VOLATILE) | |
124 | # define tsan_ld_acq(ptr) (*(ptr)) | |
125 | # define tsan_st_rel(ptr, val) (*(ptr) = (val)) | |
126 | # endif | |
ede3e665 | 127 | |
3d27ac8d | 128 | # endif |
ede3e665 | 129 | |
3d27ac8d | 130 | # ifndef TSAN_QUALIFIER |
ede3e665 | 131 | |
3d27ac8d WL |
132 | # ifdef OPENSSL_THREADS |
133 | # define TSAN_QUALIFIER volatile | |
134 | # define TSAN_REQUIRES_LOCKING | |
135 | # else /* OPENSSL_THREADS */ | |
136 | # define TSAN_QUALIFIER | |
137 | # endif /* OPENSSL_THREADS */ | |
e22cbe5e | 138 | |
3d27ac8d WL |
139 | # define tsan_load(ptr) (*(ptr)) |
140 | # define tsan_store(ptr, val) (*(ptr) = (val)) | |
141 | # define tsan_add(ptr, n) (*(ptr) += (n)) | |
a88e328c AP |
142 | /* |
143 | * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not | |
144 | * sophisticated enough to support them. Code that relies on them should be | |
145 | * protected with #ifdef tsan_ld_acq with locked fallback. | |
146 | */ | |
ede3e665 | 147 | |
3d27ac8d | 148 | # endif |
b0b456f8 | 149 | |
3d27ac8d WL |
150 | # define tsan_counter(ptr) tsan_add((ptr), 1) |
151 | # define tsan_decr(ptr) tsan_add((ptr), -1) | |
b0b456f8 | 152 | |
3d27ac8d | 153 | #endif |