]>
Commit | Line | Data |
---|---|---|
b6ab06ce | 1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
f7a9f785 | 2 | Copyright (C) 2002-2016 Free Software Foundation, Inc. |
b6ab06ce UD |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
b6ab06ce UD |
18 | |
19 | #include <assert.h> | |
20 | #include <errno.h> | |
21 | #include <libintl.h> | |
22 | #include <signal.h> | |
23 | #include <stdlib.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/param.h> | |
d8dd0080 | 26 | #include <atomic.h> |
b6ab06ce UD |
27 | |
28 | #include <tls.h> | |
11bf311e UD |
29 | #include <dl-tls.h> |
30 | #include <ldsodefs.h> | |
b6ab06ce UD |
31 | |
32 | /* Amount of excess space to allocate in the static TLS area | |
33 | to allow dynamic loading of modules defining IE-model TLS data. */ | |
11bf311e | 34 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
b6ab06ce | 35 | |
b6ab06ce UD |
36 | |
37 | /* Out-of-memory handler. */ | |
b6ab06ce UD |
38 | static void |
39 | __attribute__ ((__noreturn__)) | |
40 | oom (void) | |
41 | { | |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n"); | |
43 | } | |
b6ab06ce UD |
44 | |
45 | ||
46 | size_t | |
47 | internal_function | |
48 | _dl_next_tls_modid (void) | |
49 | { | |
50 | size_t result; | |
51 | ||
52 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) | |
53 | { | |
54 | size_t disp = 0; | |
55 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
56 | ||
57 | /* Note that this branch will never be executed during program | |
58 | start since there are no gaps at that time. Therefore it | |
59 | does not matter that the dl_tls_dtv_slotinfo is not allocated | |
60 | yet when the function is called for the first times. | |
61 | ||
62 | NB: the offset +1 is due to the fact that DTV[0] is used | |
63 | for something else. */ | |
64 | result = GL(dl_tls_static_nelem) + 1; | |
65 | if (result <= GL(dl_tls_max_dtv_idx)) | |
66 | do | |
67 | { | |
68 | while (result - disp < runp->len) | |
69 | { | |
70 | if (runp->slotinfo[result - disp].map == NULL) | |
71 | break; | |
72 | ||
73 | ++result; | |
74 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); | |
75 | } | |
76 | ||
77 | if (result - disp < runp->len) | |
78 | break; | |
79 | ||
80 | disp += runp->len; | |
81 | } | |
82 | while ((runp = runp->next) != NULL); | |
83 | ||
84 | if (result > GL(dl_tls_max_dtv_idx)) | |
85 | { | |
86 | /* The new index must indeed be exactly one higher than the | |
87 | previous high. */ | |
88 | assert (result == GL(dl_tls_max_dtv_idx) + 1); | |
89 | /* There is no gap anymore. */ | |
90 | GL(dl_tls_dtv_gaps) = false; | |
91 | ||
92 | goto nogaps; | |
93 | } | |
94 | } | |
95 | else | |
96 | { | |
97 | /* No gaps, allocate a new entry. */ | |
98 | nogaps: | |
99 | ||
100 | result = ++GL(dl_tls_max_dtv_idx); | |
101 | } | |
102 | ||
103 | return result; | |
104 | } | |
105 | ||
106 | ||
d0503676 CD |
107 | size_t |
108 | internal_function | |
109 | _dl_count_modids (void) | |
110 | { | |
111 | /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where | |
112 | we fail to load a module and unload it leaving a gap. If we don't | |
113 | have gaps then the number of modids is the current maximum so | |
114 | return that. */ | |
115 | if (__glibc_likely (!GL(dl_tls_dtv_gaps))) | |
116 | return GL(dl_tls_max_dtv_idx); | |
117 | ||
118 | /* We have gaps and are forced to count the non-NULL entries. */ | |
119 | size_t n = 0; | |
120 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
121 | while (runp != NULL) | |
122 | { | |
123 | for (size_t i = 0; i < runp->len; ++i) | |
124 | if (runp->slotinfo[i].map != NULL) | |
125 | ++n; | |
126 | ||
127 | runp = runp->next; | |
128 | } | |
129 | ||
130 | return n; | |
131 | } | |
132 | ||
133 | ||
11bf311e | 134 | #ifdef SHARED |
b6ab06ce UD |
135 | void |
136 | internal_function | |
137 | _dl_determine_tlsoffset (void) | |
138 | { | |
139 | size_t max_align = TLS_TCB_ALIGN; | |
140 | size_t freetop = 0; | |
141 | size_t freebottom = 0; | |
142 | ||
143 | /* The first element of the dtv slot info list is allocated. */ | |
144 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); | |
145 | /* There is at this point only one element in the | |
146 | dl_tls_dtv_slotinfo_list list. */ | |
147 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); | |
148 | ||
149 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; | |
150 | ||
151 | /* Determining the offset of the various parts of the static TLS | |
152 | block has several dependencies. In addition we have to work | |
153 | around bugs in some toolchains. | |
154 | ||
155 | Each TLS block from the objects available at link time has a size | |
156 | and an alignment requirement. The GNU ld computes the alignment | |
157 | requirements for the data at the positions *in the file*, though. | |
158 | I.e, it is not simply possible to allocate a block with the size | |
159 | of the TLS program header entry. The data is layed out assuming | |
160 | that the first byte of the TLS block fulfills | |
161 | ||
162 | p_vaddr mod p_align == &TLS_BLOCK mod p_align | |
163 | ||
164 | This means we have to add artificial padding at the beginning of | |
165 | the TLS block. These bytes are never used for the TLS data in | |
166 | this module but the first byte allocated must be aligned | |
167 | according to mod p_align == 0 so that the first byte of the TLS | |
168 | block is aligned according to p_vaddr mod p_align. This is ugly | |
169 | and the linker can help by computing the offsets in the TLS block | |
170 | assuming the first byte of the TLS block is aligned according to | |
171 | p_align. | |
172 | ||
173 | The extra space which might be allocated before the first byte of | |
174 | the TLS block need not go unused. The code below tries to use | |
175 | that memory for the next TLS block. This can work if the total | |
176 | memory requirement for the next TLS block is smaller than the | |
177 | gap. */ | |
178 | ||
11bf311e | 179 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
180 | /* We simply start with zero. */ |
181 | size_t offset = 0; | |
182 | ||
183 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
184 | { | |
185 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
186 | ||
187 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
188 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
189 | size_t off; | |
190 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
191 | ||
192 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) | |
193 | { | |
194 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize | |
195 | - firstbyte, slotinfo[cnt].map->l_tls_align) | |
196 | + firstbyte; | |
197 | if (off <= freebottom) | |
198 | { | |
199 | freetop = off; | |
200 | ||
201 | /* XXX For some architectures we perhaps should store the | |
202 | negative offset. */ | |
203 | slotinfo[cnt].map->l_tls_offset = off; | |
204 | continue; | |
205 | } | |
206 | } | |
207 | ||
208 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, | |
209 | slotinfo[cnt].map->l_tls_align) + firstbyte; | |
210 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize | |
211 | + (freebottom - freetop)) | |
212 | { | |
213 | freetop = offset; | |
214 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; | |
215 | } | |
216 | offset = off; | |
217 | ||
218 | /* XXX For some architectures we perhaps should store the | |
219 | negative offset. */ | |
220 | slotinfo[cnt].map->l_tls_offset = off; | |
221 | } | |
222 | ||
223 | GL(dl_tls_static_used) = offset; | |
224 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) | |
225 | + TLS_TCB_SIZE); | |
11bf311e | 226 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
227 | /* The TLS blocks start right after the TCB. */ |
228 | size_t offset = TLS_TCB_SIZE; | |
229 | ||
230 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
231 | { | |
232 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
233 | ||
234 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
235 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
236 | size_t off; | |
237 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
238 | ||
239 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) | |
240 | { | |
241 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); | |
242 | if (off - freebottom < firstbyte) | |
243 | off += slotinfo[cnt].map->l_tls_align; | |
244 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) | |
245 | { | |
246 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
247 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize | |
248 | - firstbyte); | |
249 | continue; | |
250 | } | |
251 | } | |
252 | ||
253 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); | |
254 | if (off - offset < firstbyte) | |
255 | off += slotinfo[cnt].map->l_tls_align; | |
256 | ||
257 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
258 | if (off - firstbyte - offset > freetop - freebottom) | |
259 | { | |
260 | freebottom = offset; | |
261 | freetop = off - firstbyte; | |
262 | } | |
263 | ||
264 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; | |
265 | } | |
266 | ||
267 | GL(dl_tls_static_used) = offset; | |
268 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, | |
269 | TLS_TCB_ALIGN); | |
11bf311e UD |
270 | #else |
271 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
272 | #endif | |
b6ab06ce UD |
273 | |
274 | /* The alignment requirement for the static TLS block. */ | |
275 | GL(dl_tls_static_align) = max_align; | |
276 | } | |
277 | ||
278 | ||
279 | /* This is called only when the data structure setup was skipped at startup, | |
280 | when there was no need for it then. Now we have dynamically loaded | |
281 | something needing TLS, or libpthread needs it. */ | |
282 | int | |
283 | internal_function | |
284 | _dl_tls_setup (void) | |
285 | { | |
286 | assert (GL(dl_tls_dtv_slotinfo_list) == NULL); | |
287 | assert (GL(dl_tls_max_dtv_idx) == 0); | |
288 | ||
289 | const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS; | |
290 | ||
291 | GL(dl_tls_dtv_slotinfo_list) | |
292 | = calloc (1, (sizeof (struct dtv_slotinfo_list) | |
293 | + nelem * sizeof (struct dtv_slotinfo))); | |
294 | if (GL(dl_tls_dtv_slotinfo_list) == NULL) | |
295 | return -1; | |
296 | ||
297 | GL(dl_tls_dtv_slotinfo_list)->len = nelem; | |
298 | ||
299 | /* Number of elements in the static TLS block. It can't be zero | |
300 | because of various assumptions. The one element is null. */ | |
301 | GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1; | |
302 | ||
303 | /* This initializes more variables for us. */ | |
304 | _dl_determine_tlsoffset (); | |
305 | ||
306 | return 0; | |
307 | } | |
308 | rtld_hidden_def (_dl_tls_setup) | |
11bf311e | 309 | #endif |
b6ab06ce UD |
310 | |
311 | static void * | |
312 | internal_function | |
313 | allocate_dtv (void *result) | |
314 | { | |
315 | dtv_t *dtv; | |
316 | size_t dtv_length; | |
317 | ||
318 | /* We allocate a few more elements in the dtv than are needed for the | |
319 | initial set of modules. This should avoid in most cases expansions | |
320 | of the dtv. */ | |
321 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
dd654bf9 | 322 | dtv = calloc (dtv_length + 2, sizeof (dtv_t)); |
b6ab06ce UD |
323 | if (dtv != NULL) |
324 | { | |
325 | /* This is the initial length of the dtv. */ | |
326 | dtv[0].counter = dtv_length; | |
327 | ||
328 | /* The rest of the dtv (including the generation counter) is | |
329 | Initialize with zero to indicate nothing there. */ | |
330 | ||
331 | /* Add the dtv to the thread data structures. */ | |
332 | INSTALL_DTV (result, dtv); | |
333 | } | |
334 | else | |
335 | result = NULL; | |
336 | ||
337 | return result; | |
338 | } | |
339 | ||
340 | ||
341 | /* Get size and alignment requirements of the static TLS block. */ | |
342 | void | |
343 | internal_function | |
344 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) | |
345 | { | |
346 | *sizep = GL(dl_tls_static_size); | |
347 | *alignp = GL(dl_tls_static_align); | |
348 | } | |
349 | ||
6c444ad6 FW |
350 | /* Derive the location of the pointer to the start of the original |
351 | allocation (before alignment) from the pointer to the TCB. */ | |
352 | static inline void ** | |
353 | tcb_to_pointer_to_free_location (void *tcb) | |
354 | { | |
355 | #if TLS_TCB_AT_TP | |
356 | /* The TCB follows the TLS blocks, and the pointer to the front | |
357 | follows the TCB. */ | |
358 | void **original_pointer_location = tcb + TLS_TCB_SIZE; | |
359 | #elif TLS_DTV_AT_TP | |
360 | /* The TCB comes first, preceded by the pre-TCB, and the pointer is | |
361 | before that. */ | |
362 | void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *); | |
363 | #endif | |
364 | return original_pointer_location; | |
365 | } | |
b6ab06ce UD |
366 | |
367 | void * | |
368 | internal_function | |
369 | _dl_allocate_tls_storage (void) | |
370 | { | |
371 | void *result; | |
372 | size_t size = GL(dl_tls_static_size); | |
373 | ||
11bf311e | 374 | #if TLS_DTV_AT_TP |
b6ab06ce UD |
375 | /* Memory layout is: |
376 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] | |
377 | ^ This should be returned. */ | |
6c444ad6 | 378 | size += TLS_PRE_TCB_SIZE; |
11bf311e | 379 | #endif |
b6ab06ce | 380 | |
6c444ad6 FW |
381 | /* Perform the allocation. Reserve space for the required alignment |
382 | and the pointer to the original allocation. */ | |
383 | size_t alignment = GL(dl_tls_static_align); | |
384 | void *allocated = malloc (size + alignment + sizeof (void *)); | |
385 | if (__glibc_unlikely (allocated == NULL)) | |
386 | return NULL; | |
b6ab06ce | 387 | |
6c444ad6 | 388 | /* Perform alignment and allocate the DTV. */ |
11bf311e | 389 | #if TLS_TCB_AT_TP |
6c444ad6 FW |
390 | /* The TCB follows the TLS blocks, which determine the alignment. |
391 | (TCB alignment requirements have been taken into account when | |
392 | calculating GL(dl_tls_static_align).) */ | |
393 | void *aligned = (void *) roundup ((uintptr_t) allocated, alignment); | |
394 | result = aligned + size - TLS_TCB_SIZE; | |
395 | ||
396 | /* Clear the TCB data structure. We can't ask the caller (i.e. | |
397 | libpthread) to do it, because we will initialize the DTV et al. */ | |
398 | memset (result, '\0', TLS_TCB_SIZE); | |
11bf311e | 399 | #elif TLS_DTV_AT_TP |
6c444ad6 FW |
400 | /* Pre-TCB and TCB come before the TLS blocks. The layout computed |
401 | in _dl_determine_tlsoffset assumes that the TCB is aligned to the | |
402 | TLS block alignment, and not just the TLS blocks after it. This | |
403 | can leave an unused alignment gap between the TCB and the TLS | |
404 | blocks. */ | |
405 | result = (void *) roundup | |
406 | (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated, | |
407 | alignment); | |
408 | ||
409 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before | |
410 | it. We can't ask the caller (i.e. libpthread) to do it, because | |
411 | we will initialize the DTV et al. */ | |
412 | memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); | |
11bf311e | 413 | #endif |
b6ab06ce | 414 | |
6c444ad6 FW |
415 | /* Record the value of the original pointer for later |
416 | deallocation. */ | |
417 | *tcb_to_pointer_to_free_location (result) = allocated; | |
b6ab06ce | 418 | |
6c444ad6 FW |
419 | result = allocate_dtv (result); |
420 | if (result == NULL) | |
421 | free (allocated); | |
b6ab06ce UD |
422 | return result; |
423 | } | |
424 | ||
425 | ||
d8dd0080 L |
426 | #ifndef SHARED |
427 | extern dtv_t _dl_static_dtv[]; | |
428 | # define _dl_initial_dtv (&_dl_static_dtv[1]) | |
429 | #endif | |
430 | ||
431 | static dtv_t * | |
432 | _dl_resize_dtv (dtv_t *dtv) | |
433 | { | |
434 | /* Resize the dtv. */ | |
435 | dtv_t *newp; | |
436 | /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by | |
437 | other threads concurrently. */ | |
438 | size_t newsize | |
439 | = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS; | |
440 | size_t oldsize = dtv[-1].counter; | |
441 | ||
442 | if (dtv == GL(dl_initial_dtv)) | |
443 | { | |
444 | /* This is the initial dtv that was either statically allocated in | |
445 | __libc_setup_tls or allocated during rtld startup using the | |
446 | dl-minimal.c malloc instead of the real malloc. We can't free | |
447 | it, we have to abandon the old storage. */ | |
448 | ||
449 | newp = malloc ((2 + newsize) * sizeof (dtv_t)); | |
450 | if (newp == NULL) | |
451 | oom (); | |
452 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); | |
453 | } | |
454 | else | |
455 | { | |
456 | newp = realloc (&dtv[-1], | |
457 | (2 + newsize) * sizeof (dtv_t)); | |
458 | if (newp == NULL) | |
459 | oom (); | |
460 | } | |
461 | ||
462 | newp[0].counter = newsize; | |
463 | ||
464 | /* Clear the newly allocated part. */ | |
465 | memset (newp + 2 + oldsize, '\0', | |
466 | (newsize - oldsize) * sizeof (dtv_t)); | |
467 | ||
468 | /* Return the generation counter. */ | |
469 | return &newp[1]; | |
470 | } | |
471 | ||
472 | ||
b6ab06ce UD |
473 | void * |
474 | internal_function | |
475 | _dl_allocate_tls_init (void *result) | |
476 | { | |
477 | if (result == NULL) | |
478 | /* The memory allocation failed. */ | |
479 | return NULL; | |
480 | ||
481 | dtv_t *dtv = GET_DTV (result); | |
482 | struct dtv_slotinfo_list *listp; | |
483 | size_t total = 0; | |
484 | size_t maxgen = 0; | |
485 | ||
d8dd0080 L |
486 | /* Check if the current dtv is big enough. */ |
487 | if (dtv[-1].counter < GL(dl_tls_max_dtv_idx)) | |
488 | { | |
489 | /* Resize the dtv. */ | |
490 | dtv = _dl_resize_dtv (dtv); | |
491 | ||
492 | /* Install this new dtv in the thread data structures. */ | |
493 | INSTALL_DTV (result, &dtv[-1]); | |
494 | } | |
495 | ||
b6ab06ce UD |
496 | /* We have to prepare the dtv for all currently loaded modules using |
497 | TLS. For those which are dynamically loaded we add the values | |
498 | indicating deferred allocation. */ | |
499 | listp = GL(dl_tls_dtv_slotinfo_list); | |
500 | while (1) | |
501 | { | |
502 | size_t cnt; | |
503 | ||
504 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
505 | { | |
506 | struct link_map *map; | |
507 | void *dest; | |
508 | ||
509 | /* Check for the total number of used slots. */ | |
510 | if (total + cnt > GL(dl_tls_max_dtv_idx)) | |
511 | break; | |
512 | ||
513 | map = listp->slotinfo[cnt].map; | |
514 | if (map == NULL) | |
515 | /* Unused entry. */ | |
516 | continue; | |
517 | ||
518 | /* Keep track of the maximum generation number. This might | |
519 | not be the generation counter. */ | |
d0503676 | 520 | assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation)); |
b6ab06ce UD |
521 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); |
522 | ||
f8aeae34 | 523 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; |
a2ff21f8 | 524 | dtv[map->l_tls_modid].pointer.to_free = NULL; |
f8aeae34 | 525 | |
4c533566 UD |
526 | if (map->l_tls_offset == NO_TLS_OFFSET |
527 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) | |
f8aeae34 | 528 | continue; |
b6ab06ce | 529 | |
f8aeae34 | 530 | assert (map->l_tls_modid == total + cnt); |
b6ab06ce | 531 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); |
11bf311e | 532 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
533 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
534 | dest = (char *) result - map->l_tls_offset; | |
11bf311e | 535 | #elif TLS_DTV_AT_TP |
b6ab06ce | 536 | dest = (char *) result + map->l_tls_offset; |
11bf311e UD |
537 | #else |
538 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
539 | #endif | |
b6ab06ce | 540 | |
17af5da9 AO |
541 | /* Set up the DTV entry. The simplified __tls_get_addr that |
542 | some platforms use in static programs requires it. */ | |
543 | dtv[map->l_tls_modid].pointer.val = dest; | |
544 | ||
b6ab06ce | 545 | /* Copy the initialization image and clear the BSS part. */ |
b6ab06ce UD |
546 | memset (__mempcpy (dest, map->l_tls_initimage, |
547 | map->l_tls_initimage_size), '\0', | |
548 | map->l_tls_blocksize - map->l_tls_initimage_size); | |
549 | } | |
550 | ||
551 | total += cnt; | |
552 | if (total >= GL(dl_tls_max_dtv_idx)) | |
553 | break; | |
554 | ||
555 | listp = listp->next; | |
556 | assert (listp != NULL); | |
557 | } | |
558 | ||
559 | /* The DTV version is up-to-date now. */ | |
560 | dtv[0].counter = maxgen; | |
561 | ||
562 | return result; | |
563 | } | |
564 | rtld_hidden_def (_dl_allocate_tls_init) | |
565 | ||
566 | void * | |
567 | internal_function | |
568 | _dl_allocate_tls (void *mem) | |
569 | { | |
570 | return _dl_allocate_tls_init (mem == NULL | |
571 | ? _dl_allocate_tls_storage () | |
572 | : allocate_dtv (mem)); | |
573 | } | |
574 | rtld_hidden_def (_dl_allocate_tls) | |
575 | ||
576 | ||
577 | void | |
578 | internal_function | |
579 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) | |
580 | { | |
581 | dtv_t *dtv = GET_DTV (tcb); | |
582 | ||
583 | /* We need to free the memory allocated for non-static TLS. */ | |
584 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) | |
a2ff21f8 | 585 | free (dtv[1 + cnt].pointer.to_free); |
b6ab06ce UD |
586 | |
587 | /* The array starts with dtv[-1]. */ | |
04570aaa | 588 | if (dtv != GL(dl_initial_dtv)) |
dd654bf9 | 589 | free (dtv - 1); |
b6ab06ce UD |
590 | |
591 | if (dealloc_tcb) | |
6c444ad6 | 592 | free (*tcb_to_pointer_to_free_location (tcb)); |
b6ab06ce UD |
593 | } |
594 | rtld_hidden_def (_dl_deallocate_tls) | |
595 | ||
596 | ||
11bf311e | 597 | #ifdef SHARED |
b6ab06ce UD |
598 | /* The __tls_get_addr function has two basic forms which differ in the |
599 | arguments. The IA-64 form takes two parameters, the module ID and | |
600 | offset. The form used, among others, on IA-32 takes a reference to | |
601 | a special structure which contain the same information. The second | |
602 | form seems to be more often used (in the moment) so we default to | |
603 | it. Users of the IA-64 form have to provide adequate definitions | |
604 | of the following macros. */ | |
11bf311e UD |
605 | # ifndef GET_ADDR_ARGS |
606 | # define GET_ADDR_ARGS tls_index *ti | |
27a25b6e | 607 | # define GET_ADDR_PARAM ti |
11bf311e UD |
608 | # endif |
609 | # ifndef GET_ADDR_MODULE | |
610 | # define GET_ADDR_MODULE ti->ti_module | |
611 | # endif | |
612 | # ifndef GET_ADDR_OFFSET | |
613 | # define GET_ADDR_OFFSET ti->ti_offset | |
614 | # endif | |
b6ab06ce | 615 | |
a2ff21f8 FW |
616 | /* Allocate one DTV entry. */ |
617 | static struct dtv_pointer | |
618 | allocate_dtv_entry (size_t alignment, size_t size) | |
619 | { | |
620 | if (powerof2 (alignment) && alignment <= _Alignof (max_align_t)) | |
621 | { | |
622 | /* The alignment is supported by malloc. */ | |
623 | void *ptr = malloc (size); | |
624 | return (struct dtv_pointer) { ptr, ptr }; | |
625 | } | |
b6ab06ce | 626 | |
a2ff21f8 FW |
627 | /* Emulate memalign to by manually aligning a pointer returned by |
628 | malloc. First compute the size with an overflow check. */ | |
629 | size_t alloc_size = size + alignment; | |
630 | if (alloc_size < size) | |
631 | return (struct dtv_pointer) {}; | |
632 | ||
633 | /* Perform the allocation. This is the pointer we need to free | |
634 | later. */ | |
635 | void *start = malloc (alloc_size); | |
636 | if (start == NULL) | |
637 | return (struct dtv_pointer) {}; | |
638 | ||
639 | /* Find the aligned position within the larger allocation. */ | |
640 | void *aligned = (void *) roundup ((uintptr_t) start, alignment); | |
641 | ||
642 | return (struct dtv_pointer) { .val = aligned, .to_free = start }; | |
643 | } | |
644 | ||
645 | static struct dtv_pointer | |
73d61e4f | 646 | allocate_and_init (struct link_map *map) |
b6ab06ce | 647 | { |
a2ff21f8 FW |
648 | struct dtv_pointer result = allocate_dtv_entry |
649 | (map->l_tls_align, map->l_tls_blocksize); | |
650 | if (result.val == NULL) | |
b6ab06ce UD |
651 | oom (); |
652 | ||
73d61e4f | 653 | /* Initialize the memory. */ |
a2ff21f8 FW |
654 | memset (__mempcpy (result.val, map->l_tls_initimage, |
655 | map->l_tls_initimage_size), | |
b6ab06ce UD |
656 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); |
657 | ||
a2ff21f8 | 658 | return result; |
b6ab06ce UD |
659 | } |
660 | ||
661 | ||
662 | struct link_map * | |
663 | _dl_update_slotinfo (unsigned long int req_modid) | |
664 | { | |
665 | struct link_map *the_map = NULL; | |
666 | dtv_t *dtv = THREAD_DTV (); | |
667 | ||
668 | /* The global dl_tls_dtv_slotinfo array contains for each module | |
669 | index the generation counter current when the entry was created. | |
670 | This array never shrinks so that all module indices which were | |
671 | valid at some time can be used to access it. Before the first | |
672 | use of a new module index in this function the array was extended | |
673 | appropriately. Access also does not have to be guarded against | |
674 | modifications of the array. It is assumed that pointer-size | |
675 | values can be read atomically even in SMP environments. It is | |
676 | possible that other threads at the same time dynamically load | |
677 | code and therefore add to the slotinfo list. This is a problem | |
678 | since we must not pick up any information about incomplete work. | |
679 | The solution to this is to ignore all dtv slots which were | |
680 | created after the one we are currently interested. We know that | |
681 | dynamic loading for this module is completed and this is the last | |
682 | load operation we know finished. */ | |
683 | unsigned long int idx = req_modid; | |
684 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
685 | ||
686 | while (idx >= listp->len) | |
687 | { | |
688 | idx -= listp->len; | |
689 | listp = listp->next; | |
690 | } | |
691 | ||
692 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
693 | { | |
694 | /* The generation counter for the slot is higher than what the | |
695 | current dtv implements. We have to update the whole dtv but | |
696 | only those entries with a generation counter <= the one for | |
697 | the entry we need. */ | |
698 | size_t new_gen = listp->slotinfo[idx].gen; | |
699 | size_t total = 0; | |
73d61e4f | 700 | |
b6ab06ce UD |
701 | /* We have to look through the entire dtv slotinfo list. */ |
702 | listp = GL(dl_tls_dtv_slotinfo_list); | |
703 | do | |
704 | { | |
705 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
706 | { | |
707 | size_t gen = listp->slotinfo[cnt].gen; | |
708 | ||
709 | if (gen > new_gen) | |
710 | /* This is a slot for a generation younger than the | |
711 | one we are handling now. It might be incompletely | |
712 | set up so ignore it. */ | |
713 | continue; | |
714 | ||
715 | /* If the entry is older than the current dtv layout we | |
716 | know we don't have to handle it. */ | |
717 | if (gen <= dtv[0].counter) | |
718 | continue; | |
719 | ||
720 | /* If there is no map this means the entry is empty. */ | |
721 | struct link_map *map = listp->slotinfo[cnt].map; | |
722 | if (map == NULL) | |
723 | { | |
f8aeae34 | 724 | if (dtv[-1].counter >= total + cnt) |
b6ab06ce | 725 | { |
f8aeae34 AO |
726 | /* If this modid was used at some point the memory |
727 | might still be allocated. */ | |
a2ff21f8 | 728 | free (dtv[total + cnt].pointer.to_free); |
dd654bf9 | 729 | dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; |
a2ff21f8 | 730 | dtv[total + cnt].pointer.to_free = NULL; |
b6ab06ce UD |
731 | } |
732 | ||
733 | continue; | |
734 | } | |
735 | ||
736 | /* Check whether the current dtv array is large enough. */ | |
dd654bf9 AM |
737 | size_t modid = map->l_tls_modid; |
738 | assert (total + cnt == modid); | |
b6ab06ce UD |
739 | if (dtv[-1].counter < modid) |
740 | { | |
d8dd0080 L |
741 | /* Resize the dtv. */ |
742 | dtv = _dl_resize_dtv (dtv); | |
b6ab06ce | 743 | |
d8dd0080 | 744 | assert (modid <= dtv[-1].counter); |
b6ab06ce UD |
745 | |
746 | /* Install this new dtv in the thread data | |
747 | structures. */ | |
748 | INSTALL_NEW_DTV (dtv); | |
749 | } | |
750 | ||
751 | /* If there is currently memory allocate for this | |
752 | dtv entry free it. */ | |
753 | /* XXX Ideally we will at some point create a memory | |
754 | pool. */ | |
a2ff21f8 | 755 | free (dtv[modid].pointer.to_free); |
b6ab06ce | 756 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; |
a2ff21f8 | 757 | dtv[modid].pointer.to_free = NULL; |
b6ab06ce UD |
758 | |
759 | if (modid == req_modid) | |
760 | the_map = map; | |
761 | } | |
762 | ||
763 | total += listp->len; | |
764 | } | |
765 | while ((listp = listp->next) != NULL); | |
766 | ||
767 | /* This will be the new maximum generation counter. */ | |
768 | dtv[0].counter = new_gen; | |
769 | } | |
770 | ||
771 | return the_map; | |
772 | } | |
773 | ||
774 | ||
a3636e8b UD |
775 | static void * |
776 | __attribute_noinline__ | |
27a25b6e | 777 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
a3636e8b UD |
778 | { |
779 | /* The allocation was deferred. Do it now. */ | |
780 | if (the_map == NULL) | |
781 | { | |
782 | /* Find the link map for this module. */ | |
27a25b6e | 783 | size_t idx = GET_ADDR_MODULE; |
a3636e8b UD |
784 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
785 | ||
786 | while (idx >= listp->len) | |
787 | { | |
788 | idx -= listp->len; | |
789 | listp = listp->next; | |
790 | } | |
791 | ||
792 | the_map = listp->slotinfo[idx].map; | |
793 | } | |
73d61e4f | 794 | |
73d61e4f AM |
795 | /* Make sure that, if a dlopen running in parallel forces the |
796 | variable into static storage, we'll wait until the address in the | |
797 | static TLS block is set up, and use that. If we're undecided | |
798 | yet, make sure we make the decision holding the lock as well. */ | |
f8aeae34 AO |
799 | if (__glibc_unlikely (the_map->l_tls_offset |
800 | != FORCED_DYNAMIC_TLS_OFFSET)) | |
7f507ee1 | 801 | { |
73d61e4f | 802 | __rtld_lock_lock_recursive (GL(dl_load_lock)); |
a1ffb40e | 803 | if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) |
a3636e8b | 804 | { |
73d61e4f AM |
805 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; |
806 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
807 | } | |
f8aeae34 AO |
808 | else if (__glibc_likely (the_map->l_tls_offset |
809 | != FORCED_DYNAMIC_TLS_OFFSET)) | |
73d61e4f | 810 | { |
f8aeae34 AO |
811 | #if TLS_TCB_AT_TP |
812 | void *p = (char *) THREAD_SELF - the_map->l_tls_offset; | |
813 | #elif TLS_DTV_AT_TP | |
814 | void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE; | |
815 | #else | |
816 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
817 | #endif | |
73d61e4f | 818 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
73d61e4f | 819 | |
a2ff21f8 | 820 | dtv[GET_ADDR_MODULE].pointer.to_free = NULL; |
f8aeae34 AO |
821 | dtv[GET_ADDR_MODULE].pointer.val = p; |
822 | ||
823 | return (char *) p + GET_ADDR_OFFSET; | |
a3636e8b | 824 | } |
f8aeae34 AO |
825 | else |
826 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
a3636e8b | 827 | } |
a2ff21f8 FW |
828 | struct dtv_pointer result = allocate_and_init (the_map); |
829 | dtv[GET_ADDR_MODULE].pointer = result; | |
830 | assert (result.to_free != NULL); | |
a3636e8b | 831 | |
a2ff21f8 | 832 | return (char *) result.val + GET_ADDR_OFFSET; |
27a25b6e UD |
833 | } |
834 | ||
835 | ||
836 | static struct link_map * | |
837 | __attribute_noinline__ | |
838 | update_get_addr (GET_ADDR_ARGS) | |
839 | { | |
840 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); | |
841 | dtv_t *dtv = THREAD_DTV (); | |
842 | ||
843 | void *p = dtv[GET_ADDR_MODULE].pointer.val; | |
844 | ||
a1ffb40e | 845 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e UD |
846 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); |
847 | ||
57b957eb | 848 | return (void *) p + GET_ADDR_OFFSET; |
a3636e8b UD |
849 | } |
850 | ||
050f7298 L |
851 | /* For all machines that have a non-macro version of __tls_get_addr, we |
852 | want to use rtld_hidden_proto/rtld_hidden_def in order to call the | |
853 | internal alias for __tls_get_addr from ld.so. This avoids a PLT entry | |
854 | in ld.so for __tls_get_addr. */ | |
855 | ||
856 | #ifndef __tls_get_addr | |
857 | extern void * __tls_get_addr (GET_ADDR_ARGS); | |
858 | rtld_hidden_proto (__tls_get_addr) | |
859 | rtld_hidden_def (__tls_get_addr) | |
860 | #endif | |
a3636e8b | 861 | |
b6ab06ce UD |
862 | /* The generic dynamic and local dynamic model cannot be used in |
863 | statically linked applications. */ | |
864 | void * | |
865 | __tls_get_addr (GET_ADDR_ARGS) | |
866 | { | |
867 | dtv_t *dtv = THREAD_DTV (); | |
b6ab06ce | 868 | |
a1ffb40e | 869 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
27a25b6e | 870 | return update_get_addr (GET_ADDR_PARAM); |
b6ab06ce | 871 | |
27a25b6e | 872 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
b6ab06ce | 873 | |
a1ffb40e | 874 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e | 875 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
b6ab06ce UD |
876 | |
877 | return (char *) p + GET_ADDR_OFFSET; | |
878 | } | |
11bf311e | 879 | #endif |
b6ab06ce UD |
880 | |
881 | ||
d78efd9f RM |
882 | /* Look up the module's TLS block as for __tls_get_addr, |
883 | but never touch anything. Return null if it's not allocated yet. */ | |
884 | void * | |
d78efd9f RM |
885 | _dl_tls_get_addr_soft (struct link_map *l) |
886 | { | |
a1ffb40e | 887 | if (__glibc_unlikely (l->l_tls_modid == 0)) |
d78efd9f RM |
888 | /* This module has no TLS segment. */ |
889 | return NULL; | |
890 | ||
891 | dtv_t *dtv = THREAD_DTV (); | |
a1ffb40e | 892 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
d78efd9f RM |
893 | { |
894 | /* This thread's DTV is not completely current, | |
895 | but it might already cover this module. */ | |
896 | ||
897 | if (l->l_tls_modid >= dtv[-1].counter) | |
898 | /* Nope. */ | |
899 | return NULL; | |
900 | ||
901 | size_t idx = l->l_tls_modid; | |
902 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
903 | while (idx >= listp->len) | |
904 | { | |
905 | idx -= listp->len; | |
906 | listp = listp->next; | |
907 | } | |
908 | ||
909 | /* We've reached the slot for this module. | |
910 | If its generation counter is higher than the DTV's, | |
911 | this thread does not know about this module yet. */ | |
912 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
913 | return NULL; | |
914 | } | |
915 | ||
916 | void *data = dtv[l->l_tls_modid].pointer.val; | |
a1ffb40e | 917 | if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED)) |
d78efd9f RM |
918 | /* The DTV is current, but this thread has not yet needed |
919 | to allocate this module's segment. */ | |
920 | data = NULL; | |
921 | ||
922 | return data; | |
923 | } | |
924 | ||
b6ab06ce UD |
925 | |
926 | void | |
d78efd9f | 927 | _dl_add_to_slotinfo (struct link_map *l) |
b6ab06ce UD |
928 | { |
929 | /* Now that we know the object is loaded successfully add | |
930 | modules containing TLS data to the dtv info table. We | |
931 | might have to increase its size. */ | |
932 | struct dtv_slotinfo_list *listp; | |
933 | struct dtv_slotinfo_list *prevp; | |
934 | size_t idx = l->l_tls_modid; | |
935 | ||
936 | /* Find the place in the dtv slotinfo list. */ | |
937 | listp = GL(dl_tls_dtv_slotinfo_list); | |
938 | prevp = NULL; /* Needed to shut up gcc. */ | |
939 | do | |
940 | { | |
941 | /* Does it fit in the array of this list element? */ | |
942 | if (idx < listp->len) | |
943 | break; | |
944 | idx -= listp->len; | |
945 | prevp = listp; | |
946 | listp = listp->next; | |
947 | } | |
948 | while (listp != NULL); | |
949 | ||
950 | if (listp == NULL) | |
951 | { | |
952 | /* When we come here it means we have to add a new element | |
953 | to the slotinfo list. And the new module must be in | |
954 | the first slot. */ | |
955 | assert (idx == 0); | |
956 | ||
957 | listp = prevp->next = (struct dtv_slotinfo_list *) | |
958 | malloc (sizeof (struct dtv_slotinfo_list) | |
959 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
960 | if (listp == NULL) | |
961 | { | |
962 | /* We ran out of memory. We will simply fail this | |
963 | call but don't undo anything we did so far. The | |
964 | application will crash or be terminated anyway very | |
965 | soon. */ | |
966 | ||
967 | /* We have to do this since some entries in the dtv | |
968 | slotinfo array might already point to this | |
969 | generation. */ | |
970 | ++GL(dl_tls_generation); | |
971 | ||
972 | _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\ | |
973 | cannot create TLS data structures")); | |
974 | } | |
975 | ||
976 | listp->len = TLS_SLOTINFO_SURPLUS; | |
977 | listp->next = NULL; | |
978 | memset (listp->slotinfo, '\0', | |
979 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
980 | } | |
981 | ||
982 | /* Add the information into the slotinfo data structure. */ | |
983 | listp->slotinfo[idx].map = l; | |
984 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; | |
985 | } |