]>
Commit | Line | Data |
---|---|---|
b6ab06ce | 1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
d4697bc9 | 2 | Copyright (C) 2002-2014 Free Software Foundation, Inc. |
b6ab06ce UD |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
b6ab06ce UD |
18 | |
19 | #include <assert.h> | |
20 | #include <errno.h> | |
21 | #include <libintl.h> | |
22 | #include <signal.h> | |
23 | #include <stdlib.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/param.h> | |
26 | ||
27 | #include <tls.h> | |
11bf311e UD |
28 | #include <dl-tls.h> |
29 | #include <ldsodefs.h> | |
b6ab06ce UD |
30 | |
31 | /* Amount of excess space to allocate in the static TLS area | |
32 | to allow dynamic loading of modules defining IE-model TLS data. */ | |
11bf311e | 33 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
b6ab06ce | 34 | |
b6ab06ce UD |
35 | |
36 | /* Out-of-memory handler. */ | |
11bf311e | 37 | #ifdef SHARED |
b6ab06ce UD |
38 | static void |
39 | __attribute__ ((__noreturn__)) | |
40 | oom (void) | |
41 | { | |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n"); | |
43 | } | |
11bf311e | 44 | #endif |
b6ab06ce UD |
45 | |
46 | ||
47 | size_t | |
48 | internal_function | |
49 | _dl_next_tls_modid (void) | |
50 | { | |
51 | size_t result; | |
52 | ||
53 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) | |
54 | { | |
55 | size_t disp = 0; | |
56 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); | |
57 | ||
58 | /* Note that this branch will never be executed during program | |
59 | start since there are no gaps at that time. Therefore it | |
60 | does not matter that the dl_tls_dtv_slotinfo is not allocated | |
61 | yet when the function is called for the first times. | |
62 | ||
63 | NB: the offset +1 is due to the fact that DTV[0] is used | |
64 | for something else. */ | |
65 | result = GL(dl_tls_static_nelem) + 1; | |
66 | if (result <= GL(dl_tls_max_dtv_idx)) | |
67 | do | |
68 | { | |
69 | while (result - disp < runp->len) | |
70 | { | |
71 | if (runp->slotinfo[result - disp].map == NULL) | |
72 | break; | |
73 | ||
74 | ++result; | |
75 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); | |
76 | } | |
77 | ||
78 | if (result - disp < runp->len) | |
79 | break; | |
80 | ||
81 | disp += runp->len; | |
82 | } | |
83 | while ((runp = runp->next) != NULL); | |
84 | ||
85 | if (result > GL(dl_tls_max_dtv_idx)) | |
86 | { | |
87 | /* The new index must indeed be exactly one higher than the | |
88 | previous high. */ | |
89 | assert (result == GL(dl_tls_max_dtv_idx) + 1); | |
90 | /* There is no gap anymore. */ | |
91 | GL(dl_tls_dtv_gaps) = false; | |
92 | ||
93 | goto nogaps; | |
94 | } | |
95 | } | |
96 | else | |
97 | { | |
98 | /* No gaps, allocate a new entry. */ | |
99 | nogaps: | |
100 | ||
101 | result = ++GL(dl_tls_max_dtv_idx); | |
102 | } | |
103 | ||
104 | return result; | |
105 | } | |
106 | ||
107 | ||
11bf311e | 108 | #ifdef SHARED |
b6ab06ce UD |
109 | void |
110 | internal_function | |
111 | _dl_determine_tlsoffset (void) | |
112 | { | |
113 | size_t max_align = TLS_TCB_ALIGN; | |
114 | size_t freetop = 0; | |
115 | size_t freebottom = 0; | |
116 | ||
117 | /* The first element of the dtv slot info list is allocated. */ | |
118 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); | |
119 | /* There is at this point only one element in the | |
120 | dl_tls_dtv_slotinfo_list list. */ | |
121 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); | |
122 | ||
123 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; | |
124 | ||
125 | /* Determining the offset of the various parts of the static TLS | |
126 | block has several dependencies. In addition we have to work | |
127 | around bugs in some toolchains. | |
128 | ||
129 | Each TLS block from the objects available at link time has a size | |
130 | and an alignment requirement. The GNU ld computes the alignment | |
131 | requirements for the data at the positions *in the file*, though. | |
132 | I.e, it is not simply possible to allocate a block with the size | |
133 | of the TLS program header entry. The data is layed out assuming | |
134 | that the first byte of the TLS block fulfills | |
135 | ||
136 | p_vaddr mod p_align == &TLS_BLOCK mod p_align | |
137 | ||
138 | This means we have to add artificial padding at the beginning of | |
139 | the TLS block. These bytes are never used for the TLS data in | |
140 | this module but the first byte allocated must be aligned | |
141 | according to mod p_align == 0 so that the first byte of the TLS | |
142 | block is aligned according to p_vaddr mod p_align. This is ugly | |
143 | and the linker can help by computing the offsets in the TLS block | |
144 | assuming the first byte of the TLS block is aligned according to | |
145 | p_align. | |
146 | ||
147 | The extra space which might be allocated before the first byte of | |
148 | the TLS block need not go unused. The code below tries to use | |
149 | that memory for the next TLS block. This can work if the total | |
150 | memory requirement for the next TLS block is smaller than the | |
151 | gap. */ | |
152 | ||
11bf311e | 153 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
154 | /* We simply start with zero. */ |
155 | size_t offset = 0; | |
156 | ||
157 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
158 | { | |
159 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
160 | ||
161 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
162 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
163 | size_t off; | |
164 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
165 | ||
166 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) | |
167 | { | |
168 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize | |
169 | - firstbyte, slotinfo[cnt].map->l_tls_align) | |
170 | + firstbyte; | |
171 | if (off <= freebottom) | |
172 | { | |
173 | freetop = off; | |
174 | ||
175 | /* XXX For some architectures we perhaps should store the | |
176 | negative offset. */ | |
177 | slotinfo[cnt].map->l_tls_offset = off; | |
178 | continue; | |
179 | } | |
180 | } | |
181 | ||
182 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, | |
183 | slotinfo[cnt].map->l_tls_align) + firstbyte; | |
184 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize | |
185 | + (freebottom - freetop)) | |
186 | { | |
187 | freetop = offset; | |
188 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; | |
189 | } | |
190 | offset = off; | |
191 | ||
192 | /* XXX For some architectures we perhaps should store the | |
193 | negative offset. */ | |
194 | slotinfo[cnt].map->l_tls_offset = off; | |
195 | } | |
196 | ||
197 | GL(dl_tls_static_used) = offset; | |
198 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) | |
199 | + TLS_TCB_SIZE); | |
11bf311e | 200 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
201 | /* The TLS blocks start right after the TCB. */ |
202 | size_t offset = TLS_TCB_SIZE; | |
203 | ||
204 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) | |
205 | { | |
206 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); | |
207 | ||
208 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset | |
209 | & (slotinfo[cnt].map->l_tls_align - 1)); | |
210 | size_t off; | |
211 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); | |
212 | ||
213 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) | |
214 | { | |
215 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); | |
216 | if (off - freebottom < firstbyte) | |
217 | off += slotinfo[cnt].map->l_tls_align; | |
218 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) | |
219 | { | |
220 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
221 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize | |
222 | - firstbyte); | |
223 | continue; | |
224 | } | |
225 | } | |
226 | ||
227 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); | |
228 | if (off - offset < firstbyte) | |
229 | off += slotinfo[cnt].map->l_tls_align; | |
230 | ||
231 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; | |
232 | if (off - firstbyte - offset > freetop - freebottom) | |
233 | { | |
234 | freebottom = offset; | |
235 | freetop = off - firstbyte; | |
236 | } | |
237 | ||
238 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; | |
239 | } | |
240 | ||
241 | GL(dl_tls_static_used) = offset; | |
242 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, | |
243 | TLS_TCB_ALIGN); | |
11bf311e UD |
244 | #else |
245 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
246 | #endif | |
b6ab06ce UD |
247 | |
248 | /* The alignment requirement for the static TLS block. */ | |
249 | GL(dl_tls_static_align) = max_align; | |
250 | } | |
251 | ||
252 | ||
253 | /* This is called only when the data structure setup was skipped at startup, | |
254 | when there was no need for it then. Now we have dynamically loaded | |
255 | something needing TLS, or libpthread needs it. */ | |
256 | int | |
257 | internal_function | |
258 | _dl_tls_setup (void) | |
259 | { | |
260 | assert (GL(dl_tls_dtv_slotinfo_list) == NULL); | |
261 | assert (GL(dl_tls_max_dtv_idx) == 0); | |
262 | ||
263 | const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS; | |
264 | ||
265 | GL(dl_tls_dtv_slotinfo_list) | |
266 | = calloc (1, (sizeof (struct dtv_slotinfo_list) | |
267 | + nelem * sizeof (struct dtv_slotinfo))); | |
268 | if (GL(dl_tls_dtv_slotinfo_list) == NULL) | |
269 | return -1; | |
270 | ||
271 | GL(dl_tls_dtv_slotinfo_list)->len = nelem; | |
272 | ||
273 | /* Number of elements in the static TLS block. It can't be zero | |
274 | because of various assumptions. The one element is null. */ | |
275 | GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1; | |
276 | ||
277 | /* This initializes more variables for us. */ | |
278 | _dl_determine_tlsoffset (); | |
279 | ||
280 | return 0; | |
281 | } | |
282 | rtld_hidden_def (_dl_tls_setup) | |
11bf311e | 283 | #endif |
b6ab06ce UD |
284 | |
285 | static void * | |
286 | internal_function | |
287 | allocate_dtv (void *result) | |
288 | { | |
289 | dtv_t *dtv; | |
290 | size_t dtv_length; | |
291 | ||
292 | /* We allocate a few more elements in the dtv than are needed for the | |
293 | initial set of modules. This should avoid in most cases expansions | |
294 | of the dtv. */ | |
295 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
dd654bf9 | 296 | dtv = calloc (dtv_length + 2, sizeof (dtv_t)); |
b6ab06ce UD |
297 | if (dtv != NULL) |
298 | { | |
299 | /* This is the initial length of the dtv. */ | |
300 | dtv[0].counter = dtv_length; | |
301 | ||
302 | /* The rest of the dtv (including the generation counter) is | |
303 | Initialize with zero to indicate nothing there. */ | |
304 | ||
305 | /* Add the dtv to the thread data structures. */ | |
306 | INSTALL_DTV (result, dtv); | |
307 | } | |
308 | else | |
309 | result = NULL; | |
310 | ||
311 | return result; | |
312 | } | |
313 | ||
314 | ||
315 | /* Get size and alignment requirements of the static TLS block. */ | |
316 | void | |
317 | internal_function | |
318 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) | |
319 | { | |
320 | *sizep = GL(dl_tls_static_size); | |
321 | *alignp = GL(dl_tls_static_align); | |
322 | } | |
323 | ||
324 | ||
325 | void * | |
326 | internal_function | |
327 | _dl_allocate_tls_storage (void) | |
328 | { | |
329 | void *result; | |
330 | size_t size = GL(dl_tls_static_size); | |
331 | ||
11bf311e | 332 | #if TLS_DTV_AT_TP |
b6ab06ce UD |
333 | /* Memory layout is: |
334 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] | |
335 | ^ This should be returned. */ | |
336 | size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
337 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 338 | #endif |
b6ab06ce UD |
339 | |
340 | /* Allocate a correctly aligned chunk of memory. */ | |
341 | result = __libc_memalign (GL(dl_tls_static_align), size); | |
342 | if (__builtin_expect (result != NULL, 1)) | |
343 | { | |
344 | /* Allocate the DTV. */ | |
345 | void *allocated = result; | |
346 | ||
11bf311e | 347 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
348 | /* The TCB follows the TLS blocks. */ |
349 | result = (char *) result + size - TLS_TCB_SIZE; | |
350 | ||
351 | /* Clear the TCB data structure. We can't ask the caller (i.e. | |
352 | libpthread) to do it, because we will initialize the DTV et al. */ | |
353 | memset (result, '\0', TLS_TCB_SIZE); | |
11bf311e | 354 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
355 | result = (char *) result + size - GL(dl_tls_static_size); |
356 | ||
357 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before it. | |
358 | We can't ask the caller (i.e. libpthread) to do it, because we will | |
359 | initialize the DTV et al. */ | |
360 | memset ((char *) result - TLS_PRE_TCB_SIZE, '\0', | |
361 | TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); | |
11bf311e | 362 | #endif |
b6ab06ce UD |
363 | |
364 | result = allocate_dtv (result); | |
365 | if (result == NULL) | |
366 | free (allocated); | |
367 | } | |
368 | ||
369 | return result; | |
370 | } | |
371 | ||
372 | ||
373 | void * | |
374 | internal_function | |
375 | _dl_allocate_tls_init (void *result) | |
376 | { | |
377 | if (result == NULL) | |
378 | /* The memory allocation failed. */ | |
379 | return NULL; | |
380 | ||
381 | dtv_t *dtv = GET_DTV (result); | |
382 | struct dtv_slotinfo_list *listp; | |
383 | size_t total = 0; | |
384 | size_t maxgen = 0; | |
385 | ||
386 | /* We have to prepare the dtv for all currently loaded modules using | |
387 | TLS. For those which are dynamically loaded we add the values | |
388 | indicating deferred allocation. */ | |
389 | listp = GL(dl_tls_dtv_slotinfo_list); | |
390 | while (1) | |
391 | { | |
392 | size_t cnt; | |
393 | ||
394 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
395 | { | |
396 | struct link_map *map; | |
397 | void *dest; | |
398 | ||
399 | /* Check for the total number of used slots. */ | |
400 | if (total + cnt > GL(dl_tls_max_dtv_idx)) | |
401 | break; | |
402 | ||
403 | map = listp->slotinfo[cnt].map; | |
404 | if (map == NULL) | |
405 | /* Unused entry. */ | |
406 | continue; | |
407 | ||
408 | /* Keep track of the maximum generation number. This might | |
409 | not be the generation counter. */ | |
410 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); | |
411 | ||
4c533566 UD |
412 | if (map->l_tls_offset == NO_TLS_OFFSET |
413 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) | |
b6ab06ce UD |
414 | { |
415 | /* For dynamically loaded modules we simply store | |
416 | the value indicating deferred allocation. */ | |
417 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; | |
418 | dtv[map->l_tls_modid].pointer.is_static = false; | |
419 | continue; | |
420 | } | |
421 | ||
422 | assert (map->l_tls_modid == cnt); | |
423 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); | |
11bf311e | 424 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
425 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
426 | dest = (char *) result - map->l_tls_offset; | |
11bf311e | 427 | #elif TLS_DTV_AT_TP |
b6ab06ce | 428 | dest = (char *) result + map->l_tls_offset; |
11bf311e UD |
429 | #else |
430 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" | |
431 | #endif | |
b6ab06ce UD |
432 | |
433 | /* Copy the initialization image and clear the BSS part. */ | |
434 | dtv[map->l_tls_modid].pointer.val = dest; | |
435 | dtv[map->l_tls_modid].pointer.is_static = true; | |
436 | memset (__mempcpy (dest, map->l_tls_initimage, | |
437 | map->l_tls_initimage_size), '\0', | |
438 | map->l_tls_blocksize - map->l_tls_initimage_size); | |
439 | } | |
440 | ||
441 | total += cnt; | |
442 | if (total >= GL(dl_tls_max_dtv_idx)) | |
443 | break; | |
444 | ||
445 | listp = listp->next; | |
446 | assert (listp != NULL); | |
447 | } | |
448 | ||
449 | /* The DTV version is up-to-date now. */ | |
450 | dtv[0].counter = maxgen; | |
451 | ||
452 | return result; | |
453 | } | |
454 | rtld_hidden_def (_dl_allocate_tls_init) | |
455 | ||
456 | void * | |
457 | internal_function | |
458 | _dl_allocate_tls (void *mem) | |
459 | { | |
460 | return _dl_allocate_tls_init (mem == NULL | |
461 | ? _dl_allocate_tls_storage () | |
462 | : allocate_dtv (mem)); | |
463 | } | |
464 | rtld_hidden_def (_dl_allocate_tls) | |
465 | ||
466 | ||
b80af2f4 L |
467 | #ifndef SHARED |
468 | extern dtv_t _dl_static_dtv[]; | |
04570aaa | 469 | # define _dl_initial_dtv (&_dl_static_dtv[1]) |
b80af2f4 L |
470 | #endif |
471 | ||
b6ab06ce UD |
472 | void |
473 | internal_function | |
474 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) | |
475 | { | |
476 | dtv_t *dtv = GET_DTV (tcb); | |
477 | ||
478 | /* We need to free the memory allocated for non-static TLS. */ | |
479 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) | |
480 | if (! dtv[1 + cnt].pointer.is_static | |
481 | && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED) | |
dd654bf9 | 482 | free (dtv[1 + cnt].pointer.val); |
b6ab06ce UD |
483 | |
484 | /* The array starts with dtv[-1]. */ | |
04570aaa | 485 | if (dtv != GL(dl_initial_dtv)) |
dd654bf9 | 486 | free (dtv - 1); |
b6ab06ce UD |
487 | |
488 | if (dealloc_tcb) | |
489 | { | |
11bf311e | 490 | #if TLS_TCB_AT_TP |
b6ab06ce UD |
491 | /* The TCB follows the TLS blocks. Back up to free the whole block. */ |
492 | tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE; | |
11bf311e | 493 | #elif TLS_DTV_AT_TP |
b6ab06ce UD |
494 | /* Back up the TLS_PRE_TCB_SIZE bytes. */ |
495 | tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1) | |
496 | & ~(GL(dl_tls_static_align) - 1); | |
11bf311e | 497 | #endif |
b6ab06ce UD |
498 | free (tcb); |
499 | } | |
500 | } | |
501 | rtld_hidden_def (_dl_deallocate_tls) | |
502 | ||
503 | ||
11bf311e | 504 | #ifdef SHARED |
b6ab06ce UD |
505 | /* The __tls_get_addr function has two basic forms which differ in the |
506 | arguments. The IA-64 form takes two parameters, the module ID and | |
507 | offset. The form used, among others, on IA-32 takes a reference to | |
508 | a special structure which contain the same information. The second | |
509 | form seems to be more often used (in the moment) so we default to | |
510 | it. Users of the IA-64 form have to provide adequate definitions | |
511 | of the following macros. */ | |
11bf311e UD |
512 | # ifndef GET_ADDR_ARGS |
513 | # define GET_ADDR_ARGS tls_index *ti | |
27a25b6e | 514 | # define GET_ADDR_PARAM ti |
11bf311e UD |
515 | # endif |
516 | # ifndef GET_ADDR_MODULE | |
517 | # define GET_ADDR_MODULE ti->ti_module | |
518 | # endif | |
519 | # ifndef GET_ADDR_OFFSET | |
520 | # define GET_ADDR_OFFSET ti->ti_offset | |
521 | # endif | |
b6ab06ce UD |
522 | |
523 | ||
73d61e4f AM |
524 | static void * |
525 | allocate_and_init (struct link_map *map) | |
b6ab06ce UD |
526 | { |
527 | void *newp; | |
dd654bf9 AM |
528 | |
529 | newp = __libc_memalign (map->l_tls_align, map->l_tls_blocksize); | |
b6ab06ce UD |
530 | if (newp == NULL) |
531 | oom (); | |
532 | ||
73d61e4f | 533 | /* Initialize the memory. */ |
b6ab06ce UD |
534 | memset (__mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size), |
535 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); | |
536 | ||
73d61e4f | 537 | return newp; |
b6ab06ce UD |
538 | } |
539 | ||
540 | ||
541 | struct link_map * | |
542 | _dl_update_slotinfo (unsigned long int req_modid) | |
543 | { | |
544 | struct link_map *the_map = NULL; | |
545 | dtv_t *dtv = THREAD_DTV (); | |
546 | ||
547 | /* The global dl_tls_dtv_slotinfo array contains for each module | |
548 | index the generation counter current when the entry was created. | |
549 | This array never shrinks so that all module indices which were | |
550 | valid at some time can be used to access it. Before the first | |
551 | use of a new module index in this function the array was extended | |
552 | appropriately. Access also does not have to be guarded against | |
553 | modifications of the array. It is assumed that pointer-size | |
554 | values can be read atomically even in SMP environments. It is | |
555 | possible that other threads at the same time dynamically load | |
556 | code and therefore add to the slotinfo list. This is a problem | |
557 | since we must not pick up any information about incomplete work. | |
558 | The solution to this is to ignore all dtv slots which were | |
559 | created after the one we are currently interested. We know that | |
560 | dynamic loading for this module is completed and this is the last | |
561 | load operation we know finished. */ | |
562 | unsigned long int idx = req_modid; | |
563 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
564 | ||
565 | while (idx >= listp->len) | |
566 | { | |
567 | idx -= listp->len; | |
568 | listp = listp->next; | |
569 | } | |
570 | ||
571 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
572 | { | |
573 | /* The generation counter for the slot is higher than what the | |
574 | current dtv implements. We have to update the whole dtv but | |
575 | only those entries with a generation counter <= the one for | |
576 | the entry we need. */ | |
577 | size_t new_gen = listp->slotinfo[idx].gen; | |
578 | size_t total = 0; | |
73d61e4f | 579 | |
b6ab06ce UD |
580 | /* We have to look through the entire dtv slotinfo list. */ |
581 | listp = GL(dl_tls_dtv_slotinfo_list); | |
582 | do | |
583 | { | |
584 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) | |
585 | { | |
586 | size_t gen = listp->slotinfo[cnt].gen; | |
587 | ||
588 | if (gen > new_gen) | |
589 | /* This is a slot for a generation younger than the | |
590 | one we are handling now. It might be incompletely | |
591 | set up so ignore it. */ | |
592 | continue; | |
593 | ||
594 | /* If the entry is older than the current dtv layout we | |
595 | know we don't have to handle it. */ | |
596 | if (gen <= dtv[0].counter) | |
597 | continue; | |
598 | ||
599 | /* If there is no map this means the entry is empty. */ | |
600 | struct link_map *map = listp->slotinfo[cnt].map; | |
601 | if (map == NULL) | |
602 | { | |
603 | /* If this modid was used at some point the memory | |
604 | might still be allocated. */ | |
dd654bf9 AM |
605 | if (! dtv[total + cnt].pointer.is_static |
606 | && dtv[total + cnt].pointer.val != TLS_DTV_UNALLOCATED) | |
b6ab06ce | 607 | { |
dd654bf9 AM |
608 | free (dtv[total + cnt].pointer.val); |
609 | dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; | |
b6ab06ce UD |
610 | } |
611 | ||
612 | continue; | |
613 | } | |
614 | ||
615 | /* Check whether the current dtv array is large enough. */ | |
dd654bf9 AM |
616 | size_t modid = map->l_tls_modid; |
617 | assert (total + cnt == modid); | |
b6ab06ce UD |
618 | if (dtv[-1].counter < modid) |
619 | { | |
620 | /* Reallocate the dtv. */ | |
621 | dtv_t *newp; | |
622 | size_t newsize = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; | |
623 | size_t oldsize = dtv[-1].counter; | |
624 | ||
625 | assert (map->l_tls_modid <= newsize); | |
626 | ||
627 | if (dtv == GL(dl_initial_dtv)) | |
628 | { | |
629 | /* This is the initial dtv that was allocated | |
630 | during rtld startup using the dl-minimal.c | |
dd654bf9 | 631 | malloc instead of the real malloc. We can't |
b6ab06ce UD |
632 | free it, we have to abandon the old storage. */ |
633 | ||
dd654bf9 | 634 | newp = malloc ((2 + newsize) * sizeof (dtv_t)); |
b6ab06ce UD |
635 | if (newp == NULL) |
636 | oom (); | |
292eb817 | 637 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); |
b6ab06ce UD |
638 | } |
639 | else | |
640 | { | |
dd654bf9 | 641 | newp = realloc (&dtv[-1], |
b6ab06ce UD |
642 | (2 + newsize) * sizeof (dtv_t)); |
643 | if (newp == NULL) | |
644 | oom (); | |
645 | } | |
646 | ||
647 | newp[0].counter = newsize; | |
648 | ||
649 | /* Clear the newly allocated part. */ | |
650 | memset (newp + 2 + oldsize, '\0', | |
651 | (newsize - oldsize) * sizeof (dtv_t)); | |
652 | ||
653 | /* Point dtv to the generation counter. */ | |
654 | dtv = &newp[1]; | |
655 | ||
656 | /* Install this new dtv in the thread data | |
657 | structures. */ | |
658 | INSTALL_NEW_DTV (dtv); | |
659 | } | |
660 | ||
661 | /* If there is currently memory allocate for this | |
662 | dtv entry free it. */ | |
663 | /* XXX Ideally we will at some point create a memory | |
664 | pool. */ | |
665 | if (! dtv[modid].pointer.is_static | |
666 | && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED) | |
667 | /* Note that free is called for NULL is well. We | |
668 | deallocate even if it is this dtv entry we are | |
669 | supposed to load. The reason is that we call | |
670 | memalign and not malloc. */ | |
dd654bf9 | 671 | free (dtv[modid].pointer.val); |
b6ab06ce UD |
672 | |
673 | /* This module is loaded dynamically- We defer memory | |
674 | allocation. */ | |
675 | dtv[modid].pointer.is_static = false; | |
676 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; | |
677 | ||
678 | if (modid == req_modid) | |
679 | the_map = map; | |
680 | } | |
681 | ||
682 | total += listp->len; | |
683 | } | |
684 | while ((listp = listp->next) != NULL); | |
685 | ||
686 | /* This will be the new maximum generation counter. */ | |
687 | dtv[0].counter = new_gen; | |
688 | } | |
689 | ||
690 | return the_map; | |
691 | } | |
692 | ||
693 | ||
a3636e8b UD |
694 | static void * |
695 | __attribute_noinline__ | |
27a25b6e | 696 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
a3636e8b UD |
697 | { |
698 | /* The allocation was deferred. Do it now. */ | |
699 | if (the_map == NULL) | |
700 | { | |
701 | /* Find the link map for this module. */ | |
27a25b6e | 702 | size_t idx = GET_ADDR_MODULE; |
a3636e8b UD |
703 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
704 | ||
705 | while (idx >= listp->len) | |
706 | { | |
707 | idx -= listp->len; | |
708 | listp = listp->next; | |
709 | } | |
710 | ||
711 | the_map = listp->slotinfo[idx].map; | |
712 | } | |
73d61e4f AM |
713 | |
714 | again: | |
715 | /* Make sure that, if a dlopen running in parallel forces the | |
716 | variable into static storage, we'll wait until the address in the | |
717 | static TLS block is set up, and use that. If we're undecided | |
718 | yet, make sure we make the decision holding the lock as well. */ | |
719 | if (__builtin_expect (the_map->l_tls_offset | |
720 | != FORCED_DYNAMIC_TLS_OFFSET, 0)) | |
7f507ee1 | 721 | { |
73d61e4f | 722 | __rtld_lock_lock_recursive (GL(dl_load_lock)); |
a1ffb40e | 723 | if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) |
a3636e8b | 724 | { |
73d61e4f AM |
725 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; |
726 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
727 | } | |
728 | else | |
729 | { | |
730 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); | |
731 | if (__builtin_expect (the_map->l_tls_offset | |
732 | != FORCED_DYNAMIC_TLS_OFFSET, 1)) | |
733 | { | |
734 | void *p = dtv[GET_ADDR_MODULE].pointer.val; | |
a1ffb40e | 735 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
73d61e4f AM |
736 | goto again; |
737 | ||
738 | return (char *) p + GET_ADDR_OFFSET; | |
739 | } | |
a3636e8b UD |
740 | } |
741 | } | |
73d61e4f AM |
742 | void *p = dtv[GET_ADDR_MODULE].pointer.val = allocate_and_init (the_map); |
743 | dtv[GET_ADDR_MODULE].pointer.is_static = false; | |
a3636e8b | 744 | |
73d61e4f | 745 | return (char *) p + GET_ADDR_OFFSET; |
27a25b6e UD |
746 | } |
747 | ||
748 | ||
749 | static struct link_map * | |
750 | __attribute_noinline__ | |
751 | update_get_addr (GET_ADDR_ARGS) | |
752 | { | |
753 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); | |
754 | dtv_t *dtv = THREAD_DTV (); | |
755 | ||
756 | void *p = dtv[GET_ADDR_MODULE].pointer.val; | |
757 | ||
a1ffb40e | 758 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e UD |
759 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); |
760 | ||
57b957eb | 761 | return (void *) p + GET_ADDR_OFFSET; |
a3636e8b UD |
762 | } |
763 | ||
764 | ||
b6ab06ce UD |
765 | /* The generic dynamic and local dynamic model cannot be used in |
766 | statically linked applications. */ | |
767 | void * | |
768 | __tls_get_addr (GET_ADDR_ARGS) | |
769 | { | |
770 | dtv_t *dtv = THREAD_DTV (); | |
b6ab06ce | 771 | |
a1ffb40e | 772 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
27a25b6e | 773 | return update_get_addr (GET_ADDR_PARAM); |
b6ab06ce | 774 | |
27a25b6e | 775 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
b6ab06ce | 776 | |
a1ffb40e | 777 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
27a25b6e | 778 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
b6ab06ce UD |
779 | |
780 | return (char *) p + GET_ADDR_OFFSET; | |
781 | } | |
11bf311e | 782 | #endif |
b6ab06ce UD |
783 | |
784 | ||
d78efd9f RM |
785 | /* Look up the module's TLS block as for __tls_get_addr, |
786 | but never touch anything. Return null if it's not allocated yet. */ | |
787 | void * | |
d78efd9f RM |
788 | _dl_tls_get_addr_soft (struct link_map *l) |
789 | { | |
a1ffb40e | 790 | if (__glibc_unlikely (l->l_tls_modid == 0)) |
d78efd9f RM |
791 | /* This module has no TLS segment. */ |
792 | return NULL; | |
793 | ||
794 | dtv_t *dtv = THREAD_DTV (); | |
a1ffb40e | 795 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
d78efd9f RM |
796 | { |
797 | /* This thread's DTV is not completely current, | |
798 | but it might already cover this module. */ | |
799 | ||
800 | if (l->l_tls_modid >= dtv[-1].counter) | |
801 | /* Nope. */ | |
802 | return NULL; | |
803 | ||
804 | size_t idx = l->l_tls_modid; | |
805 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); | |
806 | while (idx >= listp->len) | |
807 | { | |
808 | idx -= listp->len; | |
809 | listp = listp->next; | |
810 | } | |
811 | ||
812 | /* We've reached the slot for this module. | |
813 | If its generation counter is higher than the DTV's, | |
814 | this thread does not know about this module yet. */ | |
815 | if (dtv[0].counter < listp->slotinfo[idx].gen) | |
816 | return NULL; | |
817 | } | |
818 | ||
819 | void *data = dtv[l->l_tls_modid].pointer.val; | |
a1ffb40e | 820 | if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED)) |
d78efd9f RM |
821 | /* The DTV is current, but this thread has not yet needed |
822 | to allocate this module's segment. */ | |
823 | data = NULL; | |
824 | ||
825 | return data; | |
826 | } | |
827 | ||
b6ab06ce UD |
828 | |
829 | void | |
d78efd9f | 830 | _dl_add_to_slotinfo (struct link_map *l) |
b6ab06ce UD |
831 | { |
832 | /* Now that we know the object is loaded successfully add | |
833 | modules containing TLS data to the dtv info table. We | |
834 | might have to increase its size. */ | |
835 | struct dtv_slotinfo_list *listp; | |
836 | struct dtv_slotinfo_list *prevp; | |
837 | size_t idx = l->l_tls_modid; | |
838 | ||
839 | /* Find the place in the dtv slotinfo list. */ | |
840 | listp = GL(dl_tls_dtv_slotinfo_list); | |
841 | prevp = NULL; /* Needed to shut up gcc. */ | |
842 | do | |
843 | { | |
844 | /* Does it fit in the array of this list element? */ | |
845 | if (idx < listp->len) | |
846 | break; | |
847 | idx -= listp->len; | |
848 | prevp = listp; | |
849 | listp = listp->next; | |
850 | } | |
851 | while (listp != NULL); | |
852 | ||
853 | if (listp == NULL) | |
854 | { | |
855 | /* When we come here it means we have to add a new element | |
856 | to the slotinfo list. And the new module must be in | |
857 | the first slot. */ | |
858 | assert (idx == 0); | |
859 | ||
860 | listp = prevp->next = (struct dtv_slotinfo_list *) | |
861 | malloc (sizeof (struct dtv_slotinfo_list) | |
862 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
863 | if (listp == NULL) | |
864 | { | |
865 | /* We ran out of memory. We will simply fail this | |
866 | call but don't undo anything we did so far. The | |
867 | application will crash or be terminated anyway very | |
868 | soon. */ | |
869 | ||
870 | /* We have to do this since some entries in the dtv | |
871 | slotinfo array might already point to this | |
872 | generation. */ | |
873 | ++GL(dl_tls_generation); | |
874 | ||
875 | _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\ | |
876 | cannot create TLS data structures")); | |
877 | } | |
878 | ||
879 | listp->len = TLS_SLOTINFO_SURPLUS; | |
880 | listp->next = NULL; | |
881 | memset (listp->slotinfo, '\0', | |
882 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); | |
883 | } | |
884 | ||
885 | /* Add the information into the slotinfo data structure. */ | |
886 | listp->slotinfo[idx].map = l; | |
887 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; | |
888 | } |