]>
Commit | Line | Data |
---|---|---|
9168f220 | 1 | // GNU D Compiler emulated TLS routines. |
99dee823 | 2 | // Copyright (C) 2019-2021 Free Software Foundation, Inc. |
9168f220 JP |
3 | |
4 | // GCC is free software; you can redistribute it and/or modify it under | |
5 | // the terms of the GNU General Public License as published by the Free | |
6 | // Software Foundation; either version 3, or (at your option) any later | |
7 | // version. | |
8 | ||
9 | // GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
10 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 | // for more details. | |
13 | ||
14 | // Under Section 7 of GPL version 3, you are granted additional | |
15 | // permissions described in the GCC Runtime Library Exception, version | |
16 | // 3.1, as published by the Free Software Foundation. | |
17 | ||
18 | // You should have received a copy of the GNU General Public License and | |
19 | // a copy of the GCC Runtime Library Exception along with this program; | |
20 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
21 | // <http://www.gnu.org/licenses/>. | |
22 | ||
23 | // This code is based on the libgcc emutls.c emulated TLS support. | |
24 | ||
25 | module gcc.emutls; | |
26 | ||
27 | import core.atomic, core.stdc.stdlib, core.stdc.string, core.sync.mutex; | |
5fee5ec3 IB |
28 | import core.internal.container.array; |
29 | import core.internal.container.hashtab; | |
9168f220 JP |
30 | import core.internal.traits : classInstanceAlignment; |
31 | import gcc.builtins, gcc.gthread; | |
32 | ||
33 | version (GNU_EMUTLS): private: | |
34 | ||
35 | alias word = __builtin_machine_uint; | |
36 | alias pointer = __builtin_pointer_uint; | |
37 | alias TlsArray = Array!(void**); | |
38 | ||
39 | /* | |
40 | * TLS control data emitted by GCC for every TLS variable. | |
41 | */ | |
42 | struct __emutls_object | |
43 | { | |
44 | word size; | |
45 | word align_; | |
46 | union | |
47 | { | |
48 | pointer offset; | |
49 | void* ptr; | |
50 | } | |
51 | ||
52 | ubyte* templ; | |
53 | } | |
54 | ||
55 | // Per-thread key to obtain the per-thread TLS variable array | |
56 | __gshared __gthread_key_t emutlsKey; | |
57 | // Largest, currently assigned TLS variable offset | |
58 | __gshared pointer emutlsMaxOffset = 0; | |
59 | // Contains the size of the TLS variables (for GC) | |
60 | __gshared Array!word emutlsSizes; | |
61 | // Contains the TLS variable array for single-threaded apps | |
62 | __gshared TlsArray singleArray; | |
63 | // List of all currently alive TlsArrays (for GC) | |
64 | __gshared HashTab!(TlsArray*, TlsArray*) emutlsArrays; | |
65 | ||
66 | // emutlsMutex Mutex + @nogc handling | |
67 | enum mutexAlign = classInstanceAlignment!Mutex; | |
68 | enum mutexClassInstanceSize = __traits(classInstanceSize, Mutex); | |
69 | __gshared align(mutexAlign) void[mutexClassInstanceSize] _emutlsMutex; | |
70 | ||
71 | @property Mutex emutlsMutex() nothrow @nogc | |
72 | { | |
73 | return cast(Mutex) _emutlsMutex.ptr; | |
74 | } | |
75 | ||
76 | /* | |
77 | * Global (de)initialization functions | |
78 | */ | |
79 | extern (C) void _d_emutls_init() nothrow @nogc | |
80 | { | |
81 | memcpy(_emutlsMutex.ptr, typeid(Mutex).initializer.ptr, _emutlsMutex.length); | |
82 | (cast(Mutex) _emutlsMutex.ptr).__ctor(); | |
83 | ||
84 | if (__gthread_key_create(&emutlsKey, &emutlsDestroyThread) != 0) | |
85 | abort(); | |
86 | } | |
87 | ||
88 | __gshared __gthread_once_t initOnce = GTHREAD_ONCE_INIT; | |
89 | ||
90 | /* | |
91 | * emutls main entrypoint, called by GCC for each TLS variable access. | |
92 | */ | |
93 | extern (C) void* __emutls_get_address(shared __emutls_object* obj) nothrow @nogc | |
94 | { | |
95 | pointer offset; | |
96 | if (__gthread_active_p()) | |
97 | { | |
98 | // Obtain the offset index into the TLS array (same for all-threads) | |
99 | // for requested var. If it is unset, obtain a new offset index. | |
100 | offset = atomicLoad!(MemoryOrder.acq, pointer)(obj.offset); | |
101 | if (__builtin_expect(offset == 0, 0)) | |
102 | { | |
103 | __gthread_once(&initOnce, &_d_emutls_init); | |
104 | emutlsMutex.lock_nothrow(); | |
105 | ||
106 | offset = obj.offset; | |
107 | if (offset == 0) | |
108 | { | |
109 | offset = ++emutlsMaxOffset; | |
110 | ||
111 | emutlsSizes.ensureLength(offset); | |
112 | // Note: it's important that we copy any data from obj and | |
113 | // do not keep an reference to obj itself: If a library is | |
114 | // unloaded, its tls variables are not removed from the arrays | |
115 | // and the GC will still scan these. If we then try to reference | |
116 | // a pointer to the data segment of an unloaded library, this | |
117 | // will crash. | |
118 | emutlsSizes[offset - 1] = obj.size; | |
119 | ||
120 | atomicStore!(MemoryOrder.rel, pointer)(obj.offset, offset); | |
121 | } | |
122 | emutlsMutex.unlock_nothrow(); | |
123 | } | |
124 | } | |
125 | // For single-threaded systems, don't synchronize | |
126 | else | |
127 | { | |
128 | if (__builtin_expect(obj.offset == 0, 0)) | |
129 | { | |
130 | offset = ++emutlsMaxOffset; | |
131 | ||
132 | emutlsSizes.ensureLength(offset); | |
133 | emutlsSizes[offset - 1] = obj.size; | |
134 | ||
135 | obj.offset = offset; | |
136 | } | |
137 | } | |
138 | ||
139 | TlsArray* arr; | |
140 | if (__gthread_active_p()) | |
141 | arr = cast(TlsArray*) __gthread_getspecific(emutlsKey); | |
142 | else | |
143 | arr = &singleArray; | |
144 | ||
145 | // This will always be false for singleArray | |
146 | if (__builtin_expect(arr == null, 0)) | |
147 | { | |
148 | arr = mallocTlsArray(offset); | |
149 | __gthread_setspecific(emutlsKey, arr); | |
150 | emutlsMutex.lock_nothrow(); | |
151 | emutlsArrays[arr] = arr; | |
152 | emutlsMutex.unlock_nothrow(); | |
153 | } | |
154 | // Check if we have to grow the per-thread array | |
155 | else if (__builtin_expect(offset > arr.length, 0)) | |
156 | { | |
157 | (*arr).ensureLength(offset); | |
158 | } | |
159 | ||
160 | // Offset 0 is used as a not-initialized marker above. In the | |
161 | // TLS array, we start at 0. | |
162 | auto index = offset - 1; | |
163 | ||
164 | // Get the per-thread pointer from the TLS array | |
165 | void** ret = (*arr)[index]; | |
166 | if (__builtin_expect(ret == null, 0)) | |
167 | { | |
168 | // Initial access, have to allocate the storage | |
169 | ret = emutlsAlloc(obj); | |
170 | (*arr)[index] = ret; | |
171 | } | |
172 | ||
173 | return ret; | |
174 | } | |
175 | ||
176 | // 1:1 copy from libgcc emutls.c | |
177 | extern (C) void __emutls_register_common(__emutls_object* obj, word size, word align_, ubyte* templ) nothrow @nogc | |
178 | { | |
179 | if (obj.size < size) | |
180 | { | |
181 | obj.size = size; | |
182 | obj.templ = null; | |
183 | } | |
184 | if (obj.align_ < align_) | |
185 | obj.align_ = align_; | |
186 | if (templ && size == obj.size) | |
187 | obj.templ = templ; | |
188 | } | |
189 | ||
190 | // 1:1 copy from libgcc emutls.c | |
191 | void** emutlsAlloc(shared __emutls_object* obj) nothrow @nogc | |
192 | { | |
193 | void* ptr; | |
194 | void* ret; | |
195 | enum pointerSize = (void*).sizeof; | |
196 | ||
197 | /* We could use here posix_memalign if available and adjust | |
198 | emutls_destroy accordingly. */ | |
199 | if ((cast() obj).align_ <= pointerSize) | |
200 | { | |
201 | ptr = malloc((cast() obj).size + pointerSize); | |
202 | if (ptr == null) | |
203 | abort(); | |
204 | (cast(void**) ptr)[0] = ptr; | |
205 | ret = ptr + pointerSize; | |
206 | } | |
207 | else | |
208 | { | |
209 | ptr = malloc(obj.size + pointerSize + obj.align_ - 1); | |
210 | if (ptr == null) | |
211 | abort(); | |
212 | ret = cast(void*)((cast(pointer)(ptr + pointerSize + obj.align_ - 1)) & ~cast( | |
213 | pointer)(obj.align_ - 1)); | |
214 | (cast(void**) ret)[-1] = ptr; | |
215 | } | |
216 | ||
217 | if (obj.templ) | |
218 | memcpy(ret, cast(ubyte*) obj.templ, cast() obj.size); | |
219 | else | |
220 | memset(ret, 0, cast() obj.size); | |
221 | ||
222 | return cast(void**) ret; | |
223 | } | |
224 | ||
225 | /* | |
226 | * When a thread has finished, remove the TLS array from the GC | |
227 | * scan list emutlsArrays, free all allocated TLS variables and | |
228 | * finally free the array. | |
229 | */ | |
230 | extern (C) void emutlsDestroyThread(void* ptr) nothrow @nogc | |
231 | { | |
232 | auto arr = cast(TlsArray*) ptr; | |
9168f220 JP |
233 | |
234 | foreach (entry; *arr) | |
235 | { | |
236 | if (entry) | |
237 | free(entry[-1]); | |
238 | } | |
239 | ||
240 | free(arr); | |
241 | } | |
242 | ||
243 | /* | |
244 | * Allocate a new TLS array, set length according to offset. | |
245 | */ | |
246 | TlsArray* mallocTlsArray(pointer offset = 0) nothrow @nogc | |
247 | { | |
248 | static assert(TlsArray.alignof == (void*).alignof); | |
249 | void[] data = malloc(TlsArray.sizeof)[0 .. TlsArray.sizeof]; | |
250 | if (data.ptr == null) | |
251 | abort(); | |
252 | ||
253 | static immutable TlsArray init = TlsArray.init; | |
254 | memcpy(data.ptr, &init, data.length); | |
255 | (cast(TlsArray*) data).length = 32; | |
256 | return cast(TlsArray*) data.ptr; | |
257 | } | |
258 | ||
259 | /* | |
260 | * Make sure array is large enough to hold an entry for offset. | |
261 | * Note: the array index will be offset - 1! | |
262 | */ | |
263 | void ensureLength(Value)(ref Array!(Value) arr, size_t offset) nothrow @nogc | |
264 | { | |
265 | // index is offset-1 | |
266 | if (offset > arr.length) | |
267 | { | |
268 | auto newSize = arr.length * 2; | |
269 | if (offset > newSize) | |
270 | newSize = offset + 32; | |
271 | arr.length = newSize; | |
272 | } | |
273 | } | |
274 | ||
275 | // Public interface | |
276 | public: | |
277 | void _d_emutls_scan(scope void delegate(void* pbeg, void* pend) nothrow cb) nothrow | |
278 | { | |
279 | void scanArray(scope TlsArray* arr) nothrow | |
280 | { | |
281 | foreach (index, entry; *arr) | |
282 | { | |
283 | auto ptr = cast(void*) entry; | |
284 | if (ptr) | |
285 | cb(ptr, ptr + emutlsSizes[index]); | |
286 | } | |
287 | } | |
288 | ||
289 | __gthread_once(&initOnce, &_d_emutls_init); | |
290 | emutlsMutex.lock_nothrow(); | |
291 | // this code is effectively nothrow | |
292 | try | |
293 | { | |
294 | foreach (arr, value; emutlsArrays) | |
295 | { | |
296 | scanArray(arr); | |
297 | } | |
298 | } | |
299 | catch (Exception) | |
300 | { | |
301 | } | |
302 | emutlsMutex.unlock_nothrow(); | |
303 | scanArray(&singleArray); | |
304 | } | |
305 | ||
306 | // Call this after druntime has been unloaded | |
307 | void _d_emutls_destroy() nothrow @nogc | |
308 | { | |
9168f220 JP |
309 | (cast(Mutex) _emutlsMutex.ptr).__dtor(); |
310 | destroy(emutlsArrays); | |
311 | } |