]>
Commit | Line | Data |
---|---|---|
b6ab06ce | 1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
d4697bc9 | 2 | Copyright (C) 2002-2014 Free Software Foundation, Inc. |
b6ab06ce UD |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
b6ab06ce UD |
18 | |
19 | #include <assert.h> | |
20 | #include <errno.h> | |
21 | #include <libintl.h> | |
22 | #include <signal.h> | |
23 | #include <stdlib.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/param.h> | |
26 | ||
27 | #include <tls.h> | |
11bf311e UD |
28 | #include <dl-tls.h> |
29 | #include <ldsodefs.h> | |
b6ab06ce UD |
30 | |
31 | /* Amount of excess space to allocate in the static TLS area | |
32 | to allow dynamic loading of modules defining IE-model TLS data. */ | |
11bf311e | 33 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
b6ab06ce | 34 | |
b6ab06ce UD |
35 | |
36 | /* Out-of-memory handler. */ | |
11bf311e | 37 | #ifdef SHARED |
b6ab06ce UD |
38 | static void |
39 | __attribute__ ((__noreturn__)) | |
40 | oom (void) | |
41 | { | |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n"); | |
43 | } | |
11bf311e | 44 | #endif |
b6ab06ce UD |
45 | |
46 | ||
47 | size_t | |
48 | internal_function | |
49 | _dl_next_tls_modid (void) | |
50 | { | |
51 | size_t result; | |
52 | ||
53 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) | |
54 | { | |
55 | size_t disp = 0; | |
56 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
57 | ||
58 | /* Note that this branch will never be executed during program | |
59 | start since there are no gaps at that time. Therefore it | |
60 | does not matter that the dl_tls_dtv_slotinfo is not allocated | |
61 | yet when the function is called for the first times. | |
62 | ||
63 | NB: the offset +1 is due to the fact that DTV[0] is used | |
64 | for something else. */ | |
65 | result = GL(dl_tls_static_nelem) + 1; | |
66 | if (result <= GL(dl_tls_max_dtv_idx)) | |
67 | do | |
68 | { | |
69 | while (result - disp < runp->len) | |
70 | { | |
71 | if (runp->slotinfo[result - disp].map == NULL) | |
72 | break; | |
73 | ||
74 | ++result; | |
75 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); | |
76 | } | |
77 | ||
78 | if (result - disp < runp->len) | |
79 | break; | |
80 | ||
81 | disp += runp->len; | |
82 | } | |
83 | while ((runp = runp->next) != NULL); | |
84 | ||
85 | if (result > GL(dl_tls_max_dtv_idx)) | |
86 | { | |
87 | /* The new index must indeed be exactly one higher than the | |
88 | previous high. */ | |
89 | assert (result == GL(dl_tls_max_dtv_idx) + 1); | |
90 | /* There is no gap anymore. */ | |
91 | GL(dl_tls_dtv_gaps) = false; | |
92 | ||
93 | goto nogaps; | |
94 | } | |
95 | } | |
96 | else | |
97 | { | |
98 | /* No gaps, allocate a new entry. */ | |
99 | nogaps: | |
100 | ||
101 | result = ++GL(dl_tls_max_dtv_idx); | |
102 | } | |
103 | ||
104 | return result; | |
105 | } | |
106 | ||
107 | ||
11bf311e | 108 | #ifdef SHARED |
b6ab06ce UD |
109 | void |
110 | internal_function | |
111 | _dl_determine_tlsoffset (void) | |
112 | { | |
113 | size_t max_align = TLS_TCB_ALIGN; | |
114 | size_t freetop = 0; | |
115 | size_t freebottom = 0; | |
116 | ||
117 | /* The first element of the dtv slot info list is allocated. */ | |
118 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); | |
119 | /* There is at this point only one element in the | |
120 | dl_tls_dtv_slotinfo_list list. */ | |
121 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); | |
122 | ||
123 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; | |
124 | ||
125 | /* Determining the offset of the various parts of the static TLS | |
126 | block has several dependencies. In addition we have to work | |
127 | around bugs in some toolchains. | |
128 | ||
129 | Each TLS block from the objects available at link time has a size | |
130 | and an alignment requirement. The GNU ld computes the alignment | |
131 | requirements for the data at the positions *in the file*, though. | |
132 | I.e, it is not simply possible to allocate a block with the size | |
133 | of the TLS program header entry. The data is layed out assuming | |
134 | that the first byte of the TLS block fulfills | |
135 | ||
136 | p_vaddr mod p_align == &TLS_BLOCK mod p_align | |
137 | ||
138 | This means we have to add artificial padding at the beginning of | |
139 | the TLS block. These bytes are never used for the TLS data in | |
140 | this module but the first byte allocated must be aligned | |
141 | according to mod p_align == 0 so that the first byte of the TLS | |
142 | block is aligned according to p_vaddr mod p_align. This is ugly | |
143 | and the linker can help by computing the offsets in the TLS block | |
144 | assuming the first byte of the TLS block is aligned according to | |
145 | p_align. | |
146 | ||
147 | The extra space which might be allocated before the first byte of | |
148 | the TLS block need not go unused. The code below tries to use | |
149 | that memory for the next TLS block. This can work if the total | |
150 | memory requirement for the next TLS block is smaller than the | |
151 | gap. */ | |
152 | ||
11bf311e | 153 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
154 | /* We simply start with zero. */ |
155 | size_t offset = 0; | |
156 | ||
157 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
158 | { | |
159 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
160 | ||
161 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
162 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
163 | size_t off; | |
164 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
165 | ||
166 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) | |
167 | { | |
168 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize | |
169 | - firstbyte, slotinfo[cnt].map->l_tls_align) | |
170 | + firstbyte; | |
171 | if (off <= freebottom) | |
172 | { | |
173 | freetop = off; | |
174 | ||
175 | /* XXX For some architectures we perhaps should store the | |
176 | negative offset. */ | |
177 | slotinfo[cnt].map->l_tls_offset = off; | |
178 | continue; | |
179 | } | |
180 | } | |
181 | ||
182 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, | |
183 | slotinfo[cnt].map->l_tls_align) + firstbyte; | |
184 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize | |
185 | + (freebottom - freetop)) | |
186 | { | |
187 | freetop = offset; | |
188 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; | |
189 | } | |
190 | offset = off; | |
191 | ||
192 | /* XXX For some architectures we perhaps should store the | |
193 | negative offset. */ | |
194 | slotinfo[cnt].map->l_tls_offset = off; | |
195 | } | |
196 | ||
197 | GL(dl_tls_static_used) = offset; | |
198 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) | |
199 | + TLS_TCB_SIZE); | |
11bf311e | 200 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
201 | /* The TLS blocks start right after the TCB. */ |
202 | size_t offset = TLS_TCB_SIZE; | |
203 | ||
204 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
205 | { | |
206 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
207 | ||
208 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
209 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
210 | size_t off; | |
211 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
212 | ||
213 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) | |
214 | { | |
215 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); | |
216 | if (off - freebottom < firstbyte) | |
217 | off += slotinfo[cnt].map->l_tls_align; | |
218 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) | |
219 | { | |
220 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
221 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize | |
222 | - firstbyte); | |
223 | continue; | |
224 | } | |
225 | } | |
226 | ||
227 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); | |
228 | if (off - offset < firstbyte) | |
229 | off += slotinfo[cnt].map->l_tls_align; | |
230 | ||
231 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
232 | if (off - firstbyte - offset > freetop - freebottom) | |
233 | { | |
234 | freebottom = offset; | |
235 | freetop = off - firstbyte; | |
236 | } | |
237 | ||
238 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; | |
239 | } | |
240 | ||
241 | GL(dl_tls_static_used) = offset; | |
242 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, | |
243 | TLS_TCB_ALIGN); | |
11bf311e UD |
244 | #else |
245 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
246 | #endif | |
b6ab06ce UD |
247 | |
248 | /* The alignment requirement for the static TLS block. */ | |
249 | GL(dl_tls_static_align) = max_align; | |
250 | } | |
251 | ||
252 | ||
253 | /* This is called only when the data structure setup was skipped at startup, | |
254 | when there was no need for it then. Now we have dynamically loaded | |
255 | something needing TLS, or libpthread needs it. */ | |
256 | int | |
257 | internal_function | |
258 | _dl_tls_setup (void) | |
259 | { | |
260 | assert (GL(dl_tls_dtv_slotinfo_list) == NULL); | |
261 | assert (GL(dl_tls_max_dtv_idx) == 0); | |
262 | ||
263 | const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS; | |
264 | ||
265 | GL(dl_tls_dtv_slotinfo_list) | |
266 | = calloc (1, (sizeof (struct dtv_slotinfo_list) | |
267 | + nelem * sizeof (struct dtv_slotinfo))); | |
268 | if (GL(dl_tls_dtv_slotinfo_list) == NULL) | |
269 | return -1; | |
270 | ||
271 | GL(dl_tls_dtv_slotinfo_list)->len = nelem; | |
272 | ||
273 | /* Number of elements in the static TLS block. It can't be zero | |
274 | because of various assumptions. The one element is null. */ | |
275 | GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1; | |
276 | ||
277 | /* This initializes more variables for us. */ | |
278 | _dl_determine_tlsoffset (); | |
279 | ||
280 | return 0; | |
281 | } | |
282 | rtld_hidden_def (_dl_tls_setup) | |
11bf311e | 283 | #endif |
b6ab06ce UD |
284 | |
285 | static void * | |
286 | internal_function | |
287 | allocate_dtv (void *result) | |
288 | { | |
289 | dtv_t *dtv; | |
290 | size_t dtv_length; | |
291 | ||
292 | /* We allocate a few more elements in the dtv than are needed for the | |
293 | initial set of modules. This should avoid in most cases expansions | |
294 | of the dtv. */ | |
295 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
1f33d36a | 296 | dtv = __signal_safe_calloc (dtv_length + 2, sizeof (dtv_t)); |
b6ab06ce UD |
297 | if (dtv != NULL) |
298 | { | |
299 | /* This is the initial length of the dtv. */ | |
300 | dtv[0].counter = dtv_length; | |
301 | ||
302 | /* The rest of the dtv (including the generation counter) is | |
303 | Initialize with zero to indicate nothing there. */ | |
304 | ||
305 | /* Add the dtv to the thread data structures. */ | |
306 | INSTALL_DTV (result, dtv); | |
307 | } | |
308 | else | |
309 | result = NULL; | |
310 | ||
311 | return result; | |
312 | } | |
313 | ||
314 | ||
315 | /* Get size and alignment requirements of the static TLS block. */ | |
316 | void | |
317 | internal_function | |
318 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) | |
319 | { | |
320 | *sizep = GL(dl_tls_static_size); | |
321 | *alignp = GL(dl_tls_static_align); | |
322 | } | |
323 | ||
324 | ||
325 | void * | |
326 | internal_function | |
327 | _dl_allocate_tls_storage (void) | |
328 | { | |
329 | void *result; | |
330 | size_t size = GL(dl_tls_static_size); | |
331 | ||
11bf311e | 332 | #if TLS_DTV_AT_TP |
b6ab06ce UD |
333 | /* Memory layout is: |
334 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] | |
335 | ^ This should be returned. */ | |
336 | size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
337 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 338 | #endif |
b6ab06ce UD |
339 | |
340 | /* Allocate a correctly aligned chunk of memory. */ | |
341 | result = __libc_memalign (GL(dl_tls_static_align), size); | |
342 | if (__builtin_expect (result != NULL, 1)) | |
343 | { | |
344 | /* Allocate the DTV. */ | |
345 | void *allocated = result; | |
346 | ||
11bf311e | 347 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
348 | /* The TCB follows the TLS blocks. */ |
349 | result = (char *) result + size - TLS_TCB_SIZE; | |
350 | ||
351 | /* Clear the TCB data structure. We can't ask the caller (i.e. | |
352 | libpthread) to do it, because we will initialize the DTV et al. */ | |
353 | memset (result, '\0', TLS_TCB_SIZE); | |
11bf311e | 354 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
355 | result = (char *) result + size - GL(dl_tls_static_size); |
356 | ||
357 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before it. | |
358 | We can't ask the caller (i.e. libpthread) to do it, because we will | |
359 | initialize the DTV et al. */ | |
360 | memset ((char *) result - TLS_PRE_TCB_SIZE, '\0', | |
361 | TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); | |
11bf311e | 362 | #endif |
b6ab06ce UD |
363 | |
364 | result = allocate_dtv (result); | |
365 | if (result == NULL) | |
366 | free (allocated); | |
367 | } | |
368 | ||
369 | return result; | |
370 | } | |
371 | ||
372 | ||
373 | void * | |
374 | internal_function | |
375 | _dl_allocate_tls_init (void *result) | |
376 | { | |
377 | if (result == NULL) | |
378 | /* The memory allocation failed. */ | |
379 | return NULL; | |
380 | ||
381 | dtv_t *dtv = GET_DTV (result); | |
382 | struct dtv_slotinfo_list *listp; | |
383 | size_t total = 0; | |
384 | size_t maxgen = 0; | |
385 | ||
386 | /* We have to prepare the dtv for all currently loaded modules using | |
387 | TLS. For those which are dynamically loaded we add the values | |
388 | indicating deferred allocation. */ | |
389 | listp = GL(dl_tls_dtv_slotinfo_list); | |
390 | while (1) | |
391 | { | |
392 | size_t cnt; | |
393 | ||
394 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
395 | { | |
396 | struct link_map *map; | |
397 | void *dest; | |
398 | ||
399 | /* Check for the total number of used slots. */ | |
400 | if (total + cnt > GL(dl_tls_max_dtv_idx)) | |
401 | break; | |
402 | ||
403 | map = listp->slotinfo[cnt].map; | |
404 | if (map == NULL) | |
405 | /* Unused entry. */ | |
406 | continue; | |
407 | ||
408 | /* Keep track of the maximum generation number. This might | |
409 | not be the generation counter. */ | |
410 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); | |
411 | ||
4c533566 UD |
412 | if (map->l_tls_offset == NO_TLS_OFFSET |
413 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) | |
b6ab06ce UD |
414 | { |
415 | /* For dynamically loaded modules we simply store | |
416 | the value indicating deferred allocation. */ | |
417 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; | |
418 | dtv[map->l_tls_modid].pointer.is_static = false; | |
419 | continue; | |
420 | } | |
421 | ||
422 | assert (map->l_tls_modid == cnt); | |
423 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); | |
11bf311e | 424 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
425 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
426 | dest = (char *) result - map->l_tls_offset; | |
11bf311e | 427 | #elif TLS_DTV_AT_TP |
b6ab06ce | 428 | dest = (char *) result + map->l_tls_offset; |
11bf311e UD |
429 | #else |
430 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
431 | #endif | |
b6ab06ce UD |
432 | |
433 | /* Copy the initialization image and clear the BSS part. */ | |
434 | dtv[map->l_tls_modid].pointer.val = dest; | |
435 | dtv[map->l_tls_modid].pointer.is_static = true; | |
436 | memset (__mempcpy (dest, map->l_tls_initimage, | |
437 | map->l_tls_initimage_size), '\0', | |
438 | map->l_tls_blocksize - map->l_tls_initimage_size); | |
439 | } | |
440 | ||
441 | total += cnt; | |
442 | if (total >= GL(dl_tls_max_dtv_idx)) | |
443 | break; | |
444 | ||
445 | listp = listp->next; | |
446 | assert (listp != NULL); | |
447 | } | |
448 | ||
449 | /* The DTV version is up-to-date now. */ | |
450 | dtv[0].counter = maxgen; | |
451 | ||
452 | return result; | |
453 | } | |
454 | rtld_hidden_def (_dl_allocate_tls_init) | |
455 | ||
456 | void * | |
457 | internal_function | |
458 | _dl_allocate_tls (void *mem) | |
459 | { | |
460 | return _dl_allocate_tls_init (mem == NULL | |
461 | ? _dl_allocate_tls_storage () | |
462 | : allocate_dtv (mem)); | |
463 | } | |
464 | rtld_hidden_def (_dl_allocate_tls) | |
465 | ||
35e8f7ab PP |
466 | void |
467 | internal_function | |
468 | _dl_clear_dtv (dtv_t *dtv) | |
469 | { | |
470 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) | |
471 | if (! dtv[1 + cnt].pointer.is_static | |
472 | && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED) | |
1f33d36a | 473 | __signal_safe_free (dtv[1 + cnt].pointer.val); |
35e8f7ab PP |
474 | memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t)); |
475 | } | |
476 | ||
477 | rtld_hidden_def (_dl_clear_dtv) | |
b6ab06ce | 478 | |
b80af2f4 L |
479 | #ifndef SHARED |
480 | extern dtv_t _dl_static_dtv[]; | |
04570aaa | 481 | # define _dl_initial_dtv (&_dl_static_dtv[1]) |
b80af2f4 L |
482 | #endif |
483 | ||
b6ab06ce UD |
484 | void |
485 | internal_function | |
486 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) | |
487 | { | |
488 | dtv_t *dtv = GET_DTV (tcb); | |
489 | ||
490 | /* We need to free the memory allocated for non-static TLS. */ | |
491 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) | |
492 | if (! dtv[1 + cnt].pointer.is_static | |
493 | && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED) | |
1f33d36a | 494 | __signal_safe_free (dtv[1 + cnt].pointer.val); |
b6ab06ce UD |
495 | |
496 | /* The array starts with dtv[-1]. */ | |
04570aaa | 497 | if (dtv != GL(dl_initial_dtv)) |
1f33d36a | 498 | __signal_safe_free (dtv - 1); |
b6ab06ce UD |
499 | |
500 | if (dealloc_tcb) | |
501 | { | |
11bf311e | 502 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
503 | /* The TCB follows the TLS blocks. Back up to free the whole block. */ |
504 | tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE; | |
11bf311e | 505 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
506 | /* Back up the TLS_PRE_TCB_SIZE bytes. */ |
507 | tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
508 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 509 | #endif |
b6ab06ce UD |
510 | free (tcb); |
511 | } | |
512 | } | |
513 | rtld_hidden_def (_dl_deallocate_tls) | |
514 | ||
515 | ||
11bf311e | 516 | #ifdef SHARED |
b6ab06ce UD |
517 | /* The __tls_get_addr function has two basic forms which differ in the |
518 | arguments. The IA-64 form takes two parameters, the module ID and | |
519 | offset. The form used, among others, on IA-32 takes a reference to | |
520 | a special structure which contain the same information. The second | |
521 | form seems to be more often used (in the moment) so we default to | |
522 | it. Users of the IA-64 form have to provide adequate definitions | |
523 | of the following macros. */ | |
11bf311e UD |
524 | # ifndef GET_ADDR_ARGS |
525 | # define GET_ADDR_ARGS tls_index *ti | |
27a25b6e | 526 | # define GET_ADDR_PARAM ti |
11bf311e UD |
527 | # endif |
528 | # ifndef GET_ADDR_MODULE | |
529 | # define GET_ADDR_MODULE ti->ti_module | |
530 | # endif | |
531 | # ifndef GET_ADDR_OFFSET | |
532 | # define GET_ADDR_OFFSET ti->ti_offset | |
533 | # endif | |
b6ab06ce UD |
534 | |
535 | ||
536 | static void * | |
537 | allocate_and_init (struct link_map *map) | |
538 | { | |
539 | void *newp; | |
1f33d36a | 540 | newp = __signal_safe_memalign (map->l_tls_align, map->l_tls_blocksize); |
b6ab06ce UD |
541 | if (newp == NULL) |
542 | oom (); | |
543 | ||
544 | /* Initialize the memory. */ | |
545 | memset (__mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size), | |
546 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); | |
547 | ||
548 | return newp; | |
549 | } | |
550 | ||
551 | ||
552 | struct link_map * | |
553 | _dl_update_slotinfo (unsigned long int req_modid) | |
554 | { | |
555 | struct link_map *the_map = NULL; | |
556 | dtv_t *dtv = THREAD_DTV (); | |
557 | ||
558 | /* The global dl_tls_dtv_slotinfo array contains for each module | |
559 | index the generation counter current when the entry was created. | |
560 | This array never shrinks so that all module indices which were | |
561 | valid at some time can be used to access it. Before the first | |
562 | use of a new module index in this function the array was extended | |
563 | appropriately. Access also does not have to be guarded against | |
564 | modifications of the array. It is assumed that pointer-size | |
565 | values can be read atomically even in SMP environments. It is | |
566 | possible that other threads at the same time dynamically load | |
567 | code and therefore add to the slotinfo list. This is a problem | |
568 | since we must not pick up any information about incomplete work. | |
569 | The solution to this is to ignore all dtv slots which were | |
570 | created after the one we are currently interested. We know that | |
571 | dynamic loading for this module is completed and this is the last | |
572 | load operation we know finished. */ | |
573 | unsigned long int idx = req_modid; | |
574 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
575 | ||
576 | while (idx >= listp->len) | |
577 | { | |
578 | idx -= listp->len; | |
579 | listp = listp->next; | |
580 | } | |
581 | ||
582 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
583 | { | |
584 | /* The generation counter for the slot is higher than what the | |
585 | current dtv implements. We have to update the whole dtv but | |
586 | only those entries with a generation counter <= the one for | |
587 | the entry we need. */ | |
588 | size_t new_gen = listp->slotinfo[idx].gen; | |
589 | size_t total = 0; | |
590 | ||
591 | /* We have to look through the entire dtv slotinfo list. */ | |
592 | listp = GL(dl_tls_dtv_slotinfo_list); | |
593 | do | |
594 | { | |
595 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
596 | { | |
597 | size_t gen = listp->slotinfo[cnt].gen; | |
598 | ||
599 | if (gen > new_gen) | |
600 | /* This is a slot for a generation younger than the | |
601 | one we are handling now. It might be incompletely | |
602 | set up so ignore it. */ | |
603 | continue; | |
604 | ||
605 | /* If the entry is older than the current dtv layout we | |
606 | know we don't have to handle it. */ | |
607 | if (gen <= dtv[0].counter) | |
608 | continue; | |
609 | ||
1f33d36a PP |
610 | size_t modid = total + cnt; |
611 | ||
b6ab06ce UD |
612 | /* If there is no map this means the entry is empty. */ |
613 | struct link_map *map = listp->slotinfo[cnt].map; | |
614 | if (map == NULL) | |
615 | { | |
616 | /* If this modid was used at some point the memory | |
617 | might still be allocated. */ | |
1f33d36a PP |
618 | if (dtv[-1].counter >= modid |
619 | && !dtv[modid].pointer.is_static | |
620 | && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED) | |
b6ab06ce | 621 | { |
1f33d36a PP |
622 | __signal_safe_free (dtv[modid].pointer.val); |
623 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; | |
b6ab06ce UD |
624 | } |
625 | ||
626 | continue; | |
627 | } | |
628 | ||
1f33d36a | 629 | assert (modid == map->l_tls_modid); |
b6ab06ce | 630 | /* Check whether the current dtv array is large enough. */ |
b6ab06ce UD |
631 | if (dtv[-1].counter < modid) |
632 | { | |
633 | /* Reallocate the dtv. */ | |
634 | dtv_t *newp; | |
635 | size_t newsize = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
636 | size_t oldsize = dtv[-1].counter; | |
637 | ||
638 | assert (map->l_tls_modid <= newsize); | |
639 | ||
640 | if (dtv == GL(dl_initial_dtv)) | |
641 | { | |
642 | /* This is the initial dtv that was allocated | |
643 | during rtld startup using the dl-minimal.c | |
1f33d36a | 644 | malloc instead of the real allocator. We can't |
b6ab06ce UD |
645 | free it, we have to abandon the old storage. */ |
646 | ||
1f33d36a PP |
647 | newp = __signal_safe_malloc ( |
648 | (2 + newsize) * sizeof (dtv_t)); | |
b6ab06ce UD |
649 | if (newp == NULL) |
650 | oom (); | |
292eb817 | 651 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); |
b6ab06ce UD |
652 | } |
653 | else | |
654 | { | |
1f33d36a | 655 | newp = __signal_safe_realloc (&dtv[-1], |
b6ab06ce UD |
656 | (2 + newsize) * sizeof (dtv_t)); |
657 | if (newp == NULL) | |
658 | oom (); | |
659 | } | |
660 | ||
661 | newp[0].counter = newsize; | |
662 | ||
663 | /* Clear the newly allocated part. */ | |
664 | memset (newp + 2 + oldsize, '\0', | |
665 | (newsize - oldsize) * sizeof (dtv_t)); | |
666 | ||
667 | /* Point dtv to the generation counter. */ | |
668 | dtv = &newp[1]; | |
669 | ||
670 | /* Install this new dtv in the thread data | |
671 | structures. */ | |
672 | INSTALL_NEW_DTV (dtv); | |
673 | } | |
674 | ||
675 | /* If there is currently memory allocate for this | |
676 | dtv entry free it. */ | |
677 | /* XXX Ideally we will at some point create a memory | |
678 | pool. */ | |
679 | if (! dtv[modid].pointer.is_static | |
680 | && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED) | |
681 | /* Note that free is called for NULL is well. We | |
682 | deallocate even if it is this dtv entry we are | |
683 | supposed to load. The reason is that we call | |
684 | memalign and not malloc. */ | |
1f33d36a | 685 | __signal_safe_free (dtv[modid].pointer.val); |
b6ab06ce UD |
686 | |
687 | /* This module is loaded dynamically- We defer memory | |
688 | allocation. */ | |
689 | dtv[modid].pointer.is_static = false; | |
690 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; | |
691 | ||
692 | if (modid == req_modid) | |
693 | the_map = map; | |
694 | } | |
695 | ||
696 | total += listp->len; | |
697 | } | |
698 | while ((listp = listp->next) != NULL); | |
699 | ||
700 | /* This will be the new maximum generation counter. */ | |
701 | dtv[0].counter = new_gen; | |
702 | } | |
703 | ||
704 | return the_map; | |
705 | } | |
706 | ||
707 | ||
a3636e8b UD |
708 | static void * |
709 | __attribute_noinline__ | |
27a25b6e | 710 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
a3636e8b UD |
711 | { |
712 | /* The allocation was deferred. Do it now. */ | |
713 | if (the_map == NULL) | |
714 | { | |
715 | /* Find the link map for this module. */ | |
27a25b6e | 716 | size_t idx = GET_ADDR_MODULE; |
a3636e8b UD |
717 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
718 | ||
719 | while (idx >= listp->len) | |
720 | { | |
721 | idx -= listp->len; | |
722 | listp = listp->next; | |
723 | } | |
724 | ||
725 | the_map = listp->slotinfo[idx].map; | |
726 | } | |
727 | ||
728 | again: | |
729 | /* Make sure that, if a dlopen running in parallel forces the | |
730 | variable into static storage, we'll wait until the address in the | |
731 | static TLS block is set up, and use that. If we're undecided | |
732 | yet, make sure we make the decision holding the lock as well. */ | |
733 | if (__builtin_expect (the_map->l_tls_offset | |
734 | != FORCED_DYNAMIC_TLS_OFFSET, 0)) | |
735 | { | |
736 | __rtld_lock_lock_recursive (GL(dl_load_lock)); | |
737 | if (__builtin_expect (the_map->l_tls_offset == NO_TLS_OFFSET, 1)) | |
738 | { | |
739 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; | |
740 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
741 | } | |
742 | else | |
743 | { | |
744 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
745 | if (__builtin_expect (the_map->l_tls_offset | |
746 | != FORCED_DYNAMIC_TLS_OFFSET, 1)) | |
747 | { | |
27a25b6e | 748 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
a3636e8b UD |
749 | if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0)) |
750 | goto again; | |
751 | ||
27a25b6e | 752 | return (char *) p + GET_ADDR_OFFSET; |
a3636e8b UD |
753 | } |
754 | } | |
755 | } | |
27a25b6e UD |
756 | void *p = dtv[GET_ADDR_MODULE].pointer.val = allocate_and_init (the_map); |
757 | dtv[GET_ADDR_MODULE].pointer.is_static = false; | |
a3636e8b | 758 | |
27a25b6e UD |
759 | return (char *) p + GET_ADDR_OFFSET; |
760 | } | |
761 | ||
762 | ||
763 | static struct link_map * | |
764 | __attribute_noinline__ | |
765 | update_get_addr (GET_ADDR_ARGS) | |
766 | { | |
767 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); | |
768 | dtv_t *dtv = THREAD_DTV (); | |
769 | ||
770 | void *p = dtv[GET_ADDR_MODULE].pointer.val; | |
771 | ||
772 | if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0)) | |
773 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); | |
774 | ||
57b957eb | 775 | return (void *) p + GET_ADDR_OFFSET; |
a3636e8b UD |
776 | } |
777 | ||
778 | ||
b6ab06ce UD |
779 | /* The generic dynamic and local dynamic model cannot be used in |
780 | statically linked applications. */ | |
781 | void * | |
782 | __tls_get_addr (GET_ADDR_ARGS) | |
783 | { | |
784 | dtv_t *dtv = THREAD_DTV (); | |
b6ab06ce UD |
785 | |
786 | if (__builtin_expect (dtv[0].counter != GL(dl_tls_generation), 0)) | |
27a25b6e | 787 | return update_get_addr (GET_ADDR_PARAM); |
b6ab06ce | 788 | |
27a25b6e | 789 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
b6ab06ce UD |
790 | |
791 | if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0)) | |
27a25b6e | 792 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
b6ab06ce UD |
793 | |
794 | return (char *) p + GET_ADDR_OFFSET; | |
795 | } | |
11bf311e | 796 | #endif |
b6ab06ce UD |
797 | |
798 | ||
d78efd9f RM |
799 | /* Look up the module's TLS block as for __tls_get_addr, |
800 | but never touch anything. Return null if it's not allocated yet. */ | |
801 | void * | |
d78efd9f RM |
802 | _dl_tls_get_addr_soft (struct link_map *l) |
803 | { | |
804 | if (__builtin_expect (l->l_tls_modid == 0, 0)) | |
805 | /* This module has no TLS segment. */ | |
806 | return NULL; | |
807 | ||
808 | dtv_t *dtv = THREAD_DTV (); | |
809 | if (__builtin_expect (dtv[0].counter != GL(dl_tls_generation), 0)) | |
810 | { | |
811 | /* This thread's DTV is not completely current, | |
812 | but it might already cover this module. */ | |
813 | ||
814 | if (l->l_tls_modid >= dtv[-1].counter) | |
815 | /* Nope. */ | |
816 | return NULL; | |
817 | ||
818 | size_t idx = l->l_tls_modid; | |
819 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
820 | while (idx >= listp->len) | |
821 | { | |
822 | idx -= listp->len; | |
823 | listp = listp->next; | |
824 | } | |
825 | ||
826 | /* We've reached the slot for this module. | |
827 | If its generation counter is higher than the DTV's, | |
828 | this thread does not know about this module yet. */ | |
829 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
830 | return NULL; | |
831 | } | |
832 | ||
833 | void *data = dtv[l->l_tls_modid].pointer.val; | |
834 | if (__builtin_expect (data == TLS_DTV_UNALLOCATED, 0)) | |
835 | /* The DTV is current, but this thread has not yet needed | |
836 | to allocate this module's segment. */ | |
837 | data = NULL; | |
838 | ||
839 | return data; | |
840 | } | |
841 | ||
b6ab06ce UD |
842 | |
843 | void | |
d78efd9f | 844 | _dl_add_to_slotinfo (struct link_map *l) |
b6ab06ce UD |
845 | { |
846 | /* Now that we know the object is loaded successfully add | |
847 | modules containing TLS data to the dtv info table. We | |
848 | might have to increase its size. */ | |
849 | struct dtv_slotinfo_list *listp; | |
850 | struct dtv_slotinfo_list *prevp; | |
851 | size_t idx = l->l_tls_modid; | |
852 | ||
853 | /* Find the place in the dtv slotinfo list. */ | |
854 | listp = GL(dl_tls_dtv_slotinfo_list); | |
855 | prevp = NULL; /* Needed to shut up gcc. */ | |
856 | do | |
857 | { | |
858 | /* Does it fit in the array of this list element? */ | |
859 | if (idx < listp->len) | |
860 | break; | |
861 | idx -= listp->len; | |
862 | prevp = listp; | |
863 | listp = listp->next; | |
864 | } | |
865 | while (listp != NULL); | |
866 | ||
867 | if (listp == NULL) | |
868 | { | |
869 | /* When we come here it means we have to add a new element | |
870 | to the slotinfo list. And the new module must be in | |
871 | the first slot. */ | |
872 | assert (idx == 0); | |
873 | ||
874 | listp = prevp->next = (struct dtv_slotinfo_list *) | |
875 | malloc (sizeof (struct dtv_slotinfo_list) | |
876 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
877 | if (listp == NULL) | |
878 | { | |
879 | /* We ran out of memory. We will simply fail this | |
880 | call but don't undo anything we did so far. The | |
881 | application will crash or be terminated anyway very | |
882 | soon. */ | |
883 | ||
884 | /* We have to do this since some entries in the dtv | |
885 | slotinfo array might already point to this | |
886 | generation. */ | |
887 | ++GL(dl_tls_generation); | |
888 | ||
889 | _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\ | |
890 | cannot create TLS data structures")); | |
891 | } | |
892 | ||
893 | listp->len = TLS_SLOTINFO_SURPLUS; | |
894 | listp->next = NULL; | |
895 | memset (listp->slotinfo, '\0', | |
896 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
897 | } | |
898 | ||
899 | /* Add the information into the slotinfo data structure. */ | |
900 | listp->slotinfo[idx].map = l; | |
901 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; | |
902 | } |