]>
Commit | Line | Data |
---|---|---|
b6ab06ce | 1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
f7a9f785 | 2 | Copyright (C) 2002-2016 Free Software Foundation, Inc. |
b6ab06ce UD |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
b6ab06ce UD |
18 | |
19 | #include <assert.h> | |
20 | #include <errno.h> | |
21 | #include <libintl.h> | |
22 | #include <signal.h> | |
23 | #include <stdlib.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/param.h> | |
d8dd0080 | 26 | #include <atomic.h> |
b6ab06ce UD |
27 | |
28 | #include <tls.h> | |
11bf311e UD |
29 | #include <dl-tls.h> |
30 | #include <ldsodefs.h> | |
b6ab06ce UD |
31 | |
32 | /* Amount of excess space to allocate in the static TLS area | |
33 | to allow dynamic loading of modules defining IE-model TLS data. */ | |
11bf311e | 34 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
b6ab06ce | 35 | |
b6ab06ce UD |
36 | |
37 | /* Out-of-memory handler. */ | |
b6ab06ce UD |
38 | static void |
39 | __attribute__ ((__noreturn__)) | |
40 | oom (void) | |
41 | { | |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n"); | |
43 | } | |
b6ab06ce UD |
44 | |
45 | ||
46 | size_t | |
47 | internal_function | |
48 | _dl_next_tls_modid (void) | |
49 | { | |
50 | size_t result; | |
51 | ||
52 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) | |
53 | { | |
54 | size_t disp = 0; | |
55 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
56 | ||
57 | /* Note that this branch will never be executed during program | |
58 | start since there are no gaps at that time. Therefore it | |
59 | does not matter that the dl_tls_dtv_slotinfo is not allocated | |
60 | yet when the function is called for the first times. | |
61 | ||
62 | NB: the offset +1 is due to the fact that DTV[0] is used | |
63 | for something else. */ | |
64 | result = GL(dl_tls_static_nelem) + 1; | |
65 | if (result <= GL(dl_tls_max_dtv_idx)) | |
66 | do | |
67 | { | |
68 | while (result - disp < runp->len) | |
69 | { | |
70 | if (runp->slotinfo[result - disp].map == NULL) | |
71 | break; | |
72 | ||
73 | ++result; | |
74 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); | |
75 | } | |
76 | ||
77 | if (result - disp < runp->len) | |
78 | break; | |
79 | ||
80 | disp += runp->len; | |
81 | } | |
82 | while ((runp = runp->next) != NULL); | |
83 | ||
84 | if (result > GL(dl_tls_max_dtv_idx)) | |
85 | { | |
86 | /* The new index must indeed be exactly one higher than the | |
87 | previous high. */ | |
88 | assert (result == GL(dl_tls_max_dtv_idx) + 1); | |
89 | /* There is no gap anymore. */ | |
90 | GL(dl_tls_dtv_gaps) = false; | |
91 | ||
92 | goto nogaps; | |
93 | } | |
94 | } | |
95 | else | |
96 | { | |
97 | /* No gaps, allocate a new entry. */ | |
98 | nogaps: | |
99 | ||
100 | result = ++GL(dl_tls_max_dtv_idx); | |
101 | } | |
102 | ||
103 | return result; | |
104 | } | |
105 | ||
106 | ||
d0503676 CD |
107 | size_t |
108 | internal_function | |
109 | _dl_count_modids (void) | |
110 | { | |
111 | /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where | |
112 | we fail to load a module and unload it leaving a gap. If we don't | |
113 | have gaps then the number of modids is the current maximum so | |
114 | return that. */ | |
115 | if (__glibc_likely (!GL(dl_tls_dtv_gaps))) | |
116 | return GL(dl_tls_max_dtv_idx); | |
117 | ||
118 | /* We have gaps and are forced to count the non-NULL entries. */ | |
119 | size_t n = 0; | |
120 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
121 | while (runp != NULL) | |
122 | { | |
123 | for (size_t i = 0; i < runp->len; ++i) | |
124 | if (runp->slotinfo[i].map != NULL) | |
125 | ++n; | |
126 | ||
127 | runp = runp->next; | |
128 | } | |
129 | ||
130 | return n; | |
131 | } | |
132 | ||
133 | ||
11bf311e | 134 | #ifdef SHARED |
b6ab06ce UD |
135 | void |
136 | internal_function | |
137 | _dl_determine_tlsoffset (void) | |
138 | { | |
139 | size_t max_align = TLS_TCB_ALIGN; | |
140 | size_t freetop = 0; | |
141 | size_t freebottom = 0; | |
142 | ||
143 | /* The first element of the dtv slot info list is allocated. */ | |
144 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); | |
145 | /* There is at this point only one element in the | |
146 | dl_tls_dtv_slotinfo_list list. */ | |
147 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); | |
148 | ||
149 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; | |
150 | ||
151 | /* Determining the offset of the various parts of the static TLS | |
152 | block has several dependencies. In addition we have to work | |
153 | around bugs in some toolchains. | |
154 | ||
155 | Each TLS block from the objects available at link time has a size | |
156 | and an alignment requirement. The GNU ld computes the alignment | |
157 | requirements for the data at the positions *in the file*, though. | |
158 | I.e, it is not simply possible to allocate a block with the size | |
159 | of the TLS program header entry. The data is layed out assuming | |
160 | that the first byte of the TLS block fulfills | |
161 | ||
162 | p_vaddr mod p_align == &TLS_BLOCK mod p_align | |
163 | ||
164 | This means we have to add artificial padding at the beginning of | |
165 | the TLS block. These bytes are never used for the TLS data in | |
166 | this module but the first byte allocated must be aligned | |
167 | according to mod p_align == 0 so that the first byte of the TLS | |
168 | block is aligned according to p_vaddr mod p_align. This is ugly | |
169 | and the linker can help by computing the offsets in the TLS block | |
170 | assuming the first byte of the TLS block is aligned according to | |
171 | p_align. | |
172 | ||
173 | The extra space which might be allocated before the first byte of | |
174 | the TLS block need not go unused. The code below tries to use | |
175 | that memory for the next TLS block. This can work if the total | |
176 | memory requirement for the next TLS block is smaller than the | |
177 | gap. */ | |
178 | ||
11bf311e | 179 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
180 | /* We simply start with zero. */ |
181 | size_t offset = 0; | |
182 | ||
183 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
184 | { | |
185 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
186 | ||
187 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
188 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
189 | size_t off; | |
190 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
191 | ||
192 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) | |
193 | { | |
194 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize | |
195 | - firstbyte, slotinfo[cnt].map->l_tls_align) | |
196 | + firstbyte; | |
197 | if (off <= freebottom) | |
198 | { | |
199 | freetop = off; | |
200 | ||
201 | /* XXX For some architectures we perhaps should store the | |
202 | negative offset. */ | |
203 | slotinfo[cnt].map->l_tls_offset = off; | |
204 | continue; | |
205 | } | |
206 | } | |
207 | ||
208 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, | |
209 | slotinfo[cnt].map->l_tls_align) + firstbyte; | |
210 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize | |
211 | + (freebottom - freetop)) | |
212 | { | |
213 | freetop = offset; | |
214 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; | |
215 | } | |
216 | offset = off; | |
217 | ||
218 | /* XXX For some architectures we perhaps should store the | |
219 | negative offset. */ | |
220 | slotinfo[cnt].map->l_tls_offset = off; | |
221 | } | |
222 | ||
223 | GL(dl_tls_static_used) = offset; | |
224 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) | |
225 | + TLS_TCB_SIZE); | |
11bf311e | 226 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
227 | /* The TLS blocks start right after the TCB. */ |
228 | size_t offset = TLS_TCB_SIZE; | |
229 | ||
230 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
231 | { | |
232 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
233 | ||
234 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
235 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
236 | size_t off; | |
237 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
238 | ||
239 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) | |
240 | { | |
241 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); | |
242 | if (off - freebottom < firstbyte) | |
243 | off += slotinfo[cnt].map->l_tls_align; | |
244 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) | |
245 | { | |
246 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
247 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize | |
248 | - firstbyte); | |
249 | continue; | |
250 | } | |
251 | } | |
252 | ||
253 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); | |
254 | if (off - offset < firstbyte) | |
255 | off += slotinfo[cnt].map->l_tls_align; | |
256 | ||
257 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
258 | if (off - firstbyte - offset > freetop - freebottom) | |
259 | { | |
260 | freebottom = offset; | |
261 | freetop = off - firstbyte; | |
262 | } | |
263 | ||
264 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; | |
265 | } | |
266 | ||
267 | GL(dl_tls_static_used) = offset; | |
268 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, | |
269 | TLS_TCB_ALIGN); | |
11bf311e UD |
270 | #else |
271 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
272 | #endif | |
b6ab06ce UD |
273 | |
274 | /* The alignment requirement for the static TLS block. */ | |
275 | GL(dl_tls_static_align) = max_align; | |
276 | } | |
277 | ||
278 | ||
279 | /* This is called only when the data structure setup was skipped at startup, | |
280 | when there was no need for it then. Now we have dynamically loaded | |
281 | something needing TLS, or libpthread needs it. */ | |
282 | int | |
283 | internal_function | |
284 | _dl_tls_setup (void) | |
285 | { | |
286 | assert (GL(dl_tls_dtv_slotinfo_list) == NULL); | |
287 | assert (GL(dl_tls_max_dtv_idx) == 0); | |
288 | ||
289 | const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS; | |
290 | ||
291 | GL(dl_tls_dtv_slotinfo_list) | |
292 | = calloc (1, (sizeof (struct dtv_slotinfo_list) | |
293 | + nelem * sizeof (struct dtv_slotinfo))); | |
294 | if (GL(dl_tls_dtv_slotinfo_list) == NULL) | |
295 | return -1; | |
296 | ||
297 | GL(dl_tls_dtv_slotinfo_list)->len = nelem; | |
298 | ||
299 | /* Number of elements in the static TLS block. It can't be zero | |
300 | because of various assumptions. The one element is null. */ | |
301 | GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1; | |
302 | ||
303 | /* This initializes more variables for us. */ | |
304 | _dl_determine_tlsoffset (); | |
305 | ||
306 | return 0; | |
307 | } | |
308 | rtld_hidden_def (_dl_tls_setup) | |
11bf311e | 309 | #endif |
b6ab06ce UD |
310 | |
311 | static void * | |
312 | internal_function | |
313 | allocate_dtv (void *result) | |
314 | { | |
315 | dtv_t *dtv; | |
316 | size_t dtv_length; | |
317 | ||
318 | /* We allocate a few more elements in the dtv than are needed for the | |
319 | initial set of modules. This should avoid in most cases expansions | |
320 | of the dtv. */ | |
321 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
dd654bf9 | 322 | dtv = calloc (dtv_length + 2, sizeof (dtv_t)); |
b6ab06ce UD |
323 | if (dtv != NULL) |
324 | { | |
325 | /* This is the initial length of the dtv. */ | |
326 | dtv[0].counter = dtv_length; | |
327 | ||
328 | /* The rest of the dtv (including the generation counter) is | |
329 | Initialize with zero to indicate nothing there. */ | |
330 | ||
331 | /* Add the dtv to the thread data structures. */ | |
332 | INSTALL_DTV (result, dtv); | |
333 | } | |
334 | else | |
335 | result = NULL; | |
336 | ||
337 | return result; | |
338 | } | |
339 | ||
340 | ||
341 | /* Get size and alignment requirements of the static TLS block. */ | |
342 | void | |
343 | internal_function | |
344 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) | |
345 | { | |
346 | *sizep = GL(dl_tls_static_size); | |
347 | *alignp = GL(dl_tls_static_align); | |
348 | } | |
349 | ||
350 | ||
351 | void * | |
352 | internal_function | |
353 | _dl_allocate_tls_storage (void) | |
354 | { | |
355 | void *result; | |
356 | size_t size = GL(dl_tls_static_size); | |
357 | ||
11bf311e | 358 | #if TLS_DTV_AT_TP |
b6ab06ce UD |
359 | /* Memory layout is: |
360 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] | |
361 | ^ This should be returned. */ | |
362 | size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
363 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 364 | #endif |
b6ab06ce UD |
365 | |
366 | /* Allocate a correctly aligned chunk of memory. */ | |
367 | result = __libc_memalign (GL(dl_tls_static_align), size); | |
368 | if (__builtin_expect (result != NULL, 1)) | |
369 | { | |
370 | /* Allocate the DTV. */ | |
371 | void *allocated = result; | |
372 | ||
11bf311e | 373 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
374 | /* The TCB follows the TLS blocks. */ |
375 | result = (char *) result + size - TLS_TCB_SIZE; | |
376 | ||
377 | /* Clear the TCB data structure. We can't ask the caller (i.e. | |
378 | libpthread) to do it, because we will initialize the DTV et al. */ | |
379 | memset (result, '\0', TLS_TCB_SIZE); | |
11bf311e | 380 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
381 | result = (char *) result + size - GL(dl_tls_static_size); |
382 | ||
383 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before it. | |
384 | We can't ask the caller (i.e. libpthread) to do it, because we will | |
385 | initialize the DTV et al. */ | |
386 | memset ((char *) result - TLS_PRE_TCB_SIZE, '\0', | |
387 | TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); | |
11bf311e | 388 | #endif |
b6ab06ce UD |
389 | |
390 | result = allocate_dtv (result); | |
391 | if (result == NULL) | |
392 | free (allocated); | |
393 | } | |
394 | ||
395 | return result; | |
396 | } | |
397 | ||
398 | ||
d8dd0080 L |
399 | #ifndef SHARED |
400 | extern dtv_t _dl_static_dtv[]; | |
401 | # define _dl_initial_dtv (&_dl_static_dtv[1]) | |
402 | #endif | |
403 | ||
404 | static dtv_t * | |
405 | _dl_resize_dtv (dtv_t *dtv) | |
406 | { | |
407 | /* Resize the dtv. */ | |
408 | dtv_t *newp; | |
409 | /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by | |
410 | other threads concurrently. */ | |
411 | size_t newsize | |
412 | = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS; | |
413 | size_t oldsize = dtv[-1].counter; | |
414 | ||
415 | if (dtv == GL(dl_initial_dtv)) | |
416 | { | |
417 | /* This is the initial dtv that was either statically allocated in | |
418 | __libc_setup_tls or allocated during rtld startup using the | |
419 | dl-minimal.c malloc instead of the real malloc. We can't free | |
420 | it, we have to abandon the old storage. */ | |
421 | ||
422 | newp = malloc ((2 + newsize) * sizeof (dtv_t)); | |
423 | if (newp == NULL) | |
424 | oom (); | |
425 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); | |
426 | } | |
427 | else | |
428 | { | |
429 | newp = realloc (&dtv[-1], | |
430 | (2 + newsize) * sizeof (dtv_t)); | |
431 | if (newp == NULL) | |
432 | oom (); | |
433 | } | |
434 | ||
435 | newp[0].counter = newsize; | |
436 | ||
437 | /* Clear the newly allocated part. */ | |
438 | memset (newp + 2 + oldsize, '\0', | |
439 | (newsize - oldsize) * sizeof (dtv_t)); | |
440 | ||
441 | /* Return the generation counter. */ | |
442 | return &newp[1]; | |
443 | } | |
444 | ||
445 | ||
b6ab06ce UD |
446 | void * |
447 | internal_function | |
448 | _dl_allocate_tls_init (void *result) | |
449 | { | |
450 | if (result == NULL) | |
451 | /* The memory allocation failed. */ | |
452 | return NULL; | |
453 | ||
454 | dtv_t *dtv = GET_DTV (result); | |
455 | struct dtv_slotinfo_list *listp; | |
456 | size_t total = 0; | |
457 | size_t maxgen = 0; | |
458 | ||
d8dd0080 L |
459 | /* Check if the current dtv is big enough. */ |
460 | if (dtv[-1].counter < GL(dl_tls_max_dtv_idx)) | |
461 | { | |
462 | /* Resize the dtv. */ | |
463 | dtv = _dl_resize_dtv (dtv); | |
464 | ||
465 | /* Install this new dtv in the thread data structures. */ | |
466 | INSTALL_DTV (result, &dtv[-1]); | |
467 | } | |
468 | ||
b6ab06ce UD |
469 | /* We have to prepare the dtv for all currently loaded modules using |
470 | TLS. For those which are dynamically loaded we add the values | |
471 | indicating deferred allocation. */ | |
472 | listp = GL(dl_tls_dtv_slotinfo_list); | |
473 | while (1) | |
474 | { | |
475 | size_t cnt; | |
476 | ||
477 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
478 | { | |
479 | struct link_map *map; | |
480 | void *dest; | |
481 | ||
482 | /* Check for the total number of used slots. */ | |
483 | if (total + cnt > GL(dl_tls_max_dtv_idx)) | |
484 | break; | |
485 | ||
486 | map = listp->slotinfo[cnt].map; | |
487 | if (map == NULL) | |
488 | /* Unused entry. */ | |
489 | continue; | |
490 | ||
491 | /* Keep track of the maximum generation number. This might | |
492 | not be the generation counter. */ | |
d0503676 | 493 | assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation)); |
b6ab06ce UD |
494 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); |
495 | ||
f8aeae34 AO |
496 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; |
497 | dtv[map->l_tls_modid].pointer.is_static = false; | |
498 | ||
4c533566 UD |
499 | if (map->l_tls_offset == NO_TLS_OFFSET |
500 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) | |
f8aeae34 | 501 | continue; |
b6ab06ce | 502 | |
f8aeae34 | 503 | assert (map->l_tls_modid == total + cnt); |
b6ab06ce | 504 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); |
11bf311e | 505 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
506 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
507 | dest = (char *) result - map->l_tls_offset; | |
11bf311e | 508 | #elif TLS_DTV_AT_TP |
b6ab06ce | 509 | dest = (char *) result + map->l_tls_offset; |
11bf311e UD |
510 | #else |
511 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
512 | #endif | |
b6ab06ce UD |
513 | |
514 | /* Copy the initialization image and clear the BSS part. */ | |
b6ab06ce UD |
515 | memset (__mempcpy (dest, map->l_tls_initimage, |
516 | map->l_tls_initimage_size), '\0', | |
517 | map->l_tls_blocksize - map->l_tls_initimage_size); | |
518 | } | |
519 | ||
520 | total += cnt; | |
521 | if (total >= GL(dl_tls_max_dtv_idx)) | |
522 | break; | |
523 | ||
524 | listp = listp->next; | |
525 | assert (listp != NULL); | |
526 | } | |
527 | ||
528 | /* The DTV version is up-to-date now. */ | |
529 | dtv[0].counter = maxgen; | |
530 | ||
531 | return result; | |
532 | } | |
533 | rtld_hidden_def (_dl_allocate_tls_init) | |
534 | ||
535 | void * | |
536 | internal_function | |
537 | _dl_allocate_tls (void *mem) | |
538 | { | |
539 | return _dl_allocate_tls_init (mem == NULL | |
540 | ? _dl_allocate_tls_storage () | |
541 | : allocate_dtv (mem)); | |
542 | } | |
543 | rtld_hidden_def (_dl_allocate_tls) | |
544 | ||
545 | ||
546 | void | |
547 | internal_function | |
548 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) | |
549 | { | |
550 | dtv_t *dtv = GET_DTV (tcb); | |
551 | ||
552 | /* We need to free the memory allocated for non-static TLS. */ | |
553 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) | |
554 | if (! dtv[1 + cnt].pointer.is_static | |
555 | && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED) | |
dd654bf9 | 556 | free (dtv[1 + cnt].pointer.val); |
b6ab06ce UD |
557 | |
558 | /* The array starts with dtv[-1]. */ | |
04570aaa | 559 | if (dtv != GL(dl_initial_dtv)) |
dd654bf9 | 560 | free (dtv - 1); |
b6ab06ce UD |
561 | |
562 | if (dealloc_tcb) | |
563 | { | |
11bf311e | 564 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
565 | /* The TCB follows the TLS blocks. Back up to free the whole block. */ |
566 | tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE; | |
11bf311e | 567 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
568 | /* Back up the TLS_PRE_TCB_SIZE bytes. */ |
569 | tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
570 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 571 | #endif |
b6ab06ce UD |
572 | free (tcb); |
573 | } | |
574 | } | |
575 | rtld_hidden_def (_dl_deallocate_tls) | |
576 | ||
577 | ||
11bf311e | 578 | #ifdef SHARED |
b6ab06ce UD |
579 | /* The __tls_get_addr function has two basic forms which differ in the |
580 | arguments. The IA-64 form takes two parameters, the module ID and | |
581 | offset. The form used, among others, on IA-32 takes a reference to | |
582 | a special structure which contain the same information. The second | |
583 | form seems to be more often used (in the moment) so we default to | |
584 | it. Users of the IA-64 form have to provide adequate definitions | |
585 | of the following macros. */ | |
11bf311e UD |
586 | # ifndef GET_ADDR_ARGS |
587 | # define GET_ADDR_ARGS tls_index *ti | |
27a25b6e | 588 | # define GET_ADDR_PARAM ti |
11bf311e UD |
589 | # endif |
590 | # ifndef GET_ADDR_MODULE | |
591 | # define GET_ADDR_MODULE ti->ti_module | |
592 | # endif | |
593 | # ifndef GET_ADDR_OFFSET | |
594 | # define GET_ADDR_OFFSET ti->ti_offset | |
595 | # endif | |
b6ab06ce UD |
596 | |
597 | ||
73d61e4f AM |
598 | static void * |
599 | allocate_and_init (struct link_map *map) | |
b6ab06ce UD |
600 | { |
601 | void *newp; | |
dd654bf9 AM |
602 | |
603 | newp = __libc_memalign (map->l_tls_align, map->l_tls_blocksize); | |
b6ab06ce UD |
604 | if (newp == NULL) |
605 | oom (); | |
606 | ||
73d61e4f | 607 | /* Initialize the memory. */ |
b6ab06ce UD |
608 | memset (__mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size), |
609 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); | |
610 | ||
73d61e4f | 611 | return newp; |
b6ab06ce UD |
612 | } |
613 | ||
614 | ||
615 | struct link_map * | |
616 | _dl_update_slotinfo (unsigned long int req_modid) | |
617 | { | |
618 | struct link_map *the_map = NULL; | |
619 | dtv_t *dtv = THREAD_DTV (); | |
620 | ||
621 | /* The global dl_tls_dtv_slotinfo array contains for each module | |
622 | index the generation counter current when the entry was created. | |
623 | This array never shrinks so that all module indices which were | |
624 | valid at some time can be used to access it. Before the first | |
625 | use of a new module index in this function the array was extended | |
626 | appropriately. Access also does not have to be guarded against | |
627 | modifications of the array. It is assumed that pointer-size | |
628 | values can be read atomically even in SMP environments. It is | |
629 | possible that other threads at the same time dynamically load | |
630 | code and therefore add to the slotinfo list. This is a problem | |
631 | since we must not pick up any information about incomplete work. | |
632 | The solution to this is to ignore all dtv slots which were | |
633 | created after the one we are currently interested. We know that | |
634 | dynamic loading for this module is completed and this is the last | |
635 | load operation we know finished. */ | |
636 | unsigned long int idx = req_modid; | |
637 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
638 | ||
639 | while (idx >= listp->len) | |
640 | { | |
641 | idx -= listp->len; | |
642 | listp = listp->next; | |
643 | } | |
644 | ||
645 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
646 | { | |
647 | /* The generation counter for the slot is higher than what the | |
648 | current dtv implements. We have to update the whole dtv but | |
649 | only those entries with a generation counter <= the one for | |
650 | the entry we need. */ | |
651 | size_t new_gen = listp->slotinfo[idx].gen; | |
652 | size_t total = 0; | |
73d61e4f | 653 | |
b6ab06ce UD |
654 | /* We have to look through the entire dtv slotinfo list. */ |
655 | listp = GL(dl_tls_dtv_slotinfo_list); | |
656 | do | |
657 | { | |
658 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
659 | { | |
660 | size_t gen = listp->slotinfo[cnt].gen; | |
661 | ||
662 | if (gen > new_gen) | |
663 | /* This is a slot for a generation younger than the | |
664 | one we are handling now. It might be incompletely | |
665 | set up so ignore it. */ | |
666 | continue; | |
667 | ||
668 | /* If the entry is older than the current dtv layout we | |
669 | know we don't have to handle it. */ | |
670 | if (gen <= dtv[0].counter) | |
671 | continue; | |
672 | ||
673 | /* If there is no map this means the entry is empty. */ | |
674 | struct link_map *map = listp->slotinfo[cnt].map; | |
675 | if (map == NULL) | |
676 | { | |
f8aeae34 | 677 | if (dtv[-1].counter >= total + cnt) |
b6ab06ce | 678 | { |
f8aeae34 AO |
679 | /* If this modid was used at some point the memory |
680 | might still be allocated. */ | |
681 | if (! dtv[total + cnt].pointer.is_static | |
682 | && (dtv[total + cnt].pointer.val | |
683 | != TLS_DTV_UNALLOCATED)) | |
684 | free (dtv[total + cnt].pointer.val); | |
dd654bf9 | 685 | dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; |
f8aeae34 | 686 | dtv[total + cnt].pointer.is_static = false; |
b6ab06ce UD |
687 | } |
688 | ||
689 | continue; | |
690 | } | |
691 | ||
692 | /* Check whether the current dtv array is large enough. */ | |
dd654bf9 AM |
693 | size_t modid = map->l_tls_modid; |
694 | assert (total + cnt == modid); | |
b6ab06ce UD |
695 | if (dtv[-1].counter < modid) |
696 | { | |
d8dd0080 L |
697 | /* Resize the dtv. */ |
698 | dtv = _dl_resize_dtv (dtv); | |
b6ab06ce | 699 | |
d8dd0080 | 700 | assert (modid <= dtv[-1].counter); |
b6ab06ce UD |
701 | |
702 | /* Install this new dtv in the thread data | |
703 | structures. */ | |
704 | INSTALL_NEW_DTV (dtv); | |
705 | } | |
706 | ||
707 | /* If there is currently memory allocate for this | |
708 | dtv entry free it. */ | |
709 | /* XXX Ideally we will at some point create a memory | |
710 | pool. */ | |
711 | if (! dtv[modid].pointer.is_static | |
712 | && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED) | |
713 | /* Note that free is called for NULL is well. We | |
714 | deallocate even if it is this dtv entry we are | |
715 | supposed to load. The reason is that we call | |
716 | memalign and not malloc. */ | |
dd654bf9 | 717 | free (dtv[modid].pointer.val); |
b6ab06ce | 718 | |
b6ab06ce | 719 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; |
f8aeae34 | 720 | dtv[modid].pointer.is_static = false; |
b6ab06ce UD |
721 | |
722 | if (modid == req_modid) | |
723 | the_map = map; | |
724 | } | |
725 | ||
726 | total += listp->len; | |
727 | } | |
728 | while ((listp = listp->next) != NULL); | |
729 | ||
730 | /* This will be the new maximum generation counter. */ | |
731 | dtv[0].counter = new_gen; | |
732 | } | |
733 | ||
734 | return the_map; | |
735 | } | |
736 | ||
737 | ||
a3636e8b UD |
738 | static void * |
739 | __attribute_noinline__ | |
27a25b6e | 740 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
a3636e8b UD |
741 | { |
742 | /* The allocation was deferred. Do it now. */ | |
743 | if (the_map == NULL) | |
744 | { | |
745 | /* Find the link map for this module. */ | |
27a25b6e | 746 | size_t idx = GET_ADDR_MODULE; |
a3636e8b UD |
747 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
748 | ||
749 | while (idx >= listp->len) | |
750 | { | |
751 | idx -= listp->len; | |
752 | listp = listp->next; | |
753 | } | |
754 | ||
755 | the_map = listp->slotinfo[idx].map; | |
756 | } | |
73d61e4f | 757 | |
73d61e4f AM |
758 | /* Make sure that, if a dlopen running in parallel forces the |
759 | variable into static storage, we'll wait until the address in the | |
760 | static TLS block is set up, and use that. If we're undecided | |
761 | yet, make sure we make the decision holding the lock as well. */ | |
f8aeae34 AO |
762 | if (__glibc_unlikely (the_map->l_tls_offset |
763 | != FORCED_DYNAMIC_TLS_OFFSET)) | |
7f507ee1 | 764 | { |
73d61e4f | 765 | __rtld_lock_lock_recursive (GL(dl_load_lock)); |
a1ffb40e | 766 | if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) |
a3636e8b | 767 | { |
73d61e4f AM |
768 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; |
769 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
770 | } | |
f8aeae34 AO |
771 | else if (__glibc_likely (the_map->l_tls_offset |
772 | != FORCED_DYNAMIC_TLS_OFFSET)) | |
73d61e4f | 773 | { |
f8aeae34 AO |
774 | #if TLS_TCB_AT_TP |
775 | void *p = (char *) THREAD_SELF - the_map->l_tls_offset; | |
776 | #elif TLS_DTV_AT_TP | |
777 | void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE; | |
778 | #else | |
779 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
780 | #endif | |
73d61e4f | 781 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
73d61e4f | 782 | |
f8aeae34 AO |
783 | dtv[GET_ADDR_MODULE].pointer.is_static = true; |
784 | dtv[GET_ADDR_MODULE].pointer.val = p; | |
785 | ||
786 | return (char *) p + GET_ADDR_OFFSET; | |
a3636e8b | 787 | } |
f8aeae34 AO |
788 | else |
789 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
a3636e8b | 790 | } |
73d61e4f | 791 | void *p = dtv[GET_ADDR_MODULE].pointer.val = allocate_and_init (the_map); |
f8aeae34 | 792 | assert (!dtv[GET_ADDR_MODULE].pointer.is_static); |
a3636e8b | 793 | |
73d61e4f | 794 | return (char *) p + GET_ADDR_OFFSET; |
27a25b6e UD |
795 | } |
796 | ||
797 | ||
798 | static struct link_map * | |
799 | __attribute_noinline__ | |
800 | update_get_addr (GET_ADDR_ARGS) | |
801 | { | |
802 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); | |
803 | dtv_t *dtv = THREAD_DTV (); | |
804 | ||
805 | void *p = dtv[GET_ADDR_MODULE].pointer.val; | |
806 | ||
a1ffb40e | 807 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e UD |
808 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); |
809 | ||
57b957eb | 810 | return (void *) p + GET_ADDR_OFFSET; |
a3636e8b UD |
811 | } |
812 | ||
050f7298 L |
813 | /* For all machines that have a non-macro version of __tls_get_addr, we |
814 | want to use rtld_hidden_proto/rtld_hidden_def in order to call the | |
815 | internal alias for __tls_get_addr from ld.so. This avoids a PLT entry | |
816 | in ld.so for __tls_get_addr. */ | |
817 | ||
818 | #ifndef __tls_get_addr | |
819 | extern void * __tls_get_addr (GET_ADDR_ARGS); | |
820 | rtld_hidden_proto (__tls_get_addr) | |
821 | rtld_hidden_def (__tls_get_addr) | |
822 | #endif | |
a3636e8b | 823 | |
b6ab06ce UD |
824 | /* The generic dynamic and local dynamic model cannot be used in |
825 | statically linked applications. */ | |
826 | void * | |
827 | __tls_get_addr (GET_ADDR_ARGS) | |
828 | { | |
829 | dtv_t *dtv = THREAD_DTV (); | |
b6ab06ce | 830 | |
a1ffb40e | 831 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
27a25b6e | 832 | return update_get_addr (GET_ADDR_PARAM); |
b6ab06ce | 833 | |
27a25b6e | 834 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
b6ab06ce | 835 | |
a1ffb40e | 836 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e | 837 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
b6ab06ce UD |
838 | |
839 | return (char *) p + GET_ADDR_OFFSET; | |
840 | } | |
11bf311e | 841 | #endif |
b6ab06ce UD |
842 | |
843 | ||
d78efd9f RM |
844 | /* Look up the module's TLS block as for __tls_get_addr, |
845 | but never touch anything. Return null if it's not allocated yet. */ | |
846 | void * | |
d78efd9f RM |
847 | _dl_tls_get_addr_soft (struct link_map *l) |
848 | { | |
a1ffb40e | 849 | if (__glibc_unlikely (l->l_tls_modid == 0)) |
d78efd9f RM |
850 | /* This module has no TLS segment. */ |
851 | return NULL; | |
852 | ||
853 | dtv_t *dtv = THREAD_DTV (); | |
a1ffb40e | 854 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
d78efd9f RM |
855 | { |
856 | /* This thread's DTV is not completely current, | |
857 | but it might already cover this module. */ | |
858 | ||
859 | if (l->l_tls_modid >= dtv[-1].counter) | |
860 | /* Nope. */ | |
861 | return NULL; | |
862 | ||
863 | size_t idx = l->l_tls_modid; | |
864 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
865 | while (idx >= listp->len) | |
866 | { | |
867 | idx -= listp->len; | |
868 | listp = listp->next; | |
869 | } | |
870 | ||
871 | /* We've reached the slot for this module. | |
872 | If its generation counter is higher than the DTV's, | |
873 | this thread does not know about this module yet. */ | |
874 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
875 | return NULL; | |
876 | } | |
877 | ||
878 | void *data = dtv[l->l_tls_modid].pointer.val; | |
a1ffb40e | 879 | if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED)) |
d78efd9f RM |
880 | /* The DTV is current, but this thread has not yet needed |
881 | to allocate this module's segment. */ | |
882 | data = NULL; | |
883 | ||
884 | return data; | |
885 | } | |
886 | ||
b6ab06ce UD |
887 | |
888 | void | |
d78efd9f | 889 | _dl_add_to_slotinfo (struct link_map *l) |
b6ab06ce UD |
890 | { |
891 | /* Now that we know the object is loaded successfully add | |
892 | modules containing TLS data to the dtv info table. We | |
893 | might have to increase its size. */ | |
894 | struct dtv_slotinfo_list *listp; | |
895 | struct dtv_slotinfo_list *prevp; | |
896 | size_t idx = l->l_tls_modid; | |
897 | ||
898 | /* Find the place in the dtv slotinfo list. */ | |
899 | listp = GL(dl_tls_dtv_slotinfo_list); | |
900 | prevp = NULL; /* Needed to shut up gcc. */ | |
901 | do | |
902 | { | |
903 | /* Does it fit in the array of this list element? */ | |
904 | if (idx < listp->len) | |
905 | break; | |
906 | idx -= listp->len; | |
907 | prevp = listp; | |
908 | listp = listp->next; | |
909 | } | |
910 | while (listp != NULL); | |
911 | ||
912 | if (listp == NULL) | |
913 | { | |
914 | /* When we come here it means we have to add a new element | |
915 | to the slotinfo list. And the new module must be in | |
916 | the first slot. */ | |
917 | assert (idx == 0); | |
918 | ||
919 | listp = prevp->next = (struct dtv_slotinfo_list *) | |
920 | malloc (sizeof (struct dtv_slotinfo_list) | |
921 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
922 | if (listp == NULL) | |
923 | { | |
924 | /* We ran out of memory. We will simply fail this | |
925 | call but don't undo anything we did so far. The | |
926 | application will crash or be terminated anyway very | |
927 | soon. */ | |
928 | ||
929 | /* We have to do this since some entries in the dtv | |
930 | slotinfo array might already point to this | |
931 | generation. */ | |
932 | ++GL(dl_tls_generation); | |
933 | ||
934 | _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\ | |
935 | cannot create TLS data structures")); | |
936 | } | |
937 | ||
938 | listp->len = TLS_SLOTINFO_SURPLUS; | |
939 | listp->next = NULL; | |
940 | memset (listp->slotinfo, '\0', | |
941 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
942 | } | |
943 | ||
944 | /* Add the information into the slotinfo data structure. */ | |
945 | listp->slotinfo[idx].map = l; | |
946 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; | |
947 | } |