]>
Commit | Line | Data |
---|---|---|
a3931336 | 1 | /* Definition for thread-local data handling. nptl/x86_64 version. |
d4697bc9 | 2 | Copyright (C) 2002-2014 Free Software Foundation, Inc. |
a3931336 UD |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
a3931336 UD |
18 | |
19 | #ifndef _TLS_H | |
20 | #define _TLS_H 1 | |
21 | ||
22 | #ifndef __ASSEMBLER__ | |
16a1d952 | 23 | # include <asm/prctl.h> /* For ARCH_SET_FS. */ |
9dcafc55 | 24 | # include <stdbool.h> |
a3931336 UD |
25 | # include <stddef.h> |
26 | # include <stdint.h> | |
33b5d0cc | 27 | # include <stdlib.h> |
df94b641 | 28 | # include <sysdep.h> |
c515fb51 | 29 | # include <libc-internal.h> |
5a8075b1 | 30 | # include <kernel-features.h> |
f62c8abc L |
31 | |
32 | /* Replacement type for __m128 since this file is included by ld.so, | |
33 | which is compiled with -mno-sse. It must not change the alignment | |
34 | of rtld_savespace_sse. */ | |
35 | typedef struct | |
36 | { | |
37 | int i[4]; | |
38 | } __128bits; | |
a3931336 UD |
39 | |
40 | ||
41 | /* Type for the dtv. */ | |
42 | typedef union dtv | |
43 | { | |
44 | size_t counter; | |
9dcafc55 UD |
45 | struct |
46 | { | |
47 | void *val; | |
48 | bool is_static; | |
49 | } pointer; | |
a3931336 UD |
50 | } dtv_t; |
51 | ||
52 | ||
53 | typedef struct | |
54 | { | |
46d51e9b | 55 | void *tcb; /* Pointer to the TCB. Not necessarily the |
a3931336 UD |
56 | thread descriptor used by libpthread. */ |
57 | dtv_t *dtv; | |
58 | void *self; /* Pointer to the thread descriptor. */ | |
bbde8527 | 59 | int multiple_threads; |
df94b641 | 60 | int gscope_flag; |
35f1e827 UD |
61 | uintptr_t sysinfo; |
62 | uintptr_t stack_guard; | |
827b7087 | 63 | uintptr_t pointer_guard; |
46d51e9b | 64 | unsigned long int vgetcpu_cache[2]; |
b48a267b | 65 | # ifndef __ASSUME_PRIVATE_FUTEX |
5a8075b1 | 66 | int private_futex; |
b48a267b | 67 | # else |
d1d9eaf4 | 68 | int __glibc_reserved1; |
b48a267b | 69 | # endif |
b48a267b | 70 | int rtld_must_xmm_save; |
71bb2639 | 71 | /* Reservation of some values for the TM ABI. */ |
ecbf4342 AJ |
72 | void *__private_tm[4]; |
73 | /* GCC split stack support. */ | |
74 | void *__private_ss; | |
d1d9eaf4 | 75 | long int __glibc_reserved2; |
b48a267b | 76 | /* Have space for the post-AVX register size. */ |
f62c8abc | 77 | __128bits rtld_savespace_sse[8][4] __attribute__ ((aligned (32))); |
b48a267b UD |
78 | |
79 | void *__padding[8]; | |
a3931336 | 80 | } tcbhead_t; |
bbde8527 | 81 | |
d0369fb8 UD |
82 | #else /* __ASSEMBLER__ */ |
83 | # include <tcb-offsets.h> | |
a3931336 UD |
84 | #endif |
85 | ||
86 | ||
a3931336 UD |
87 | /* Alignment requirement for the stack. */ |
88 | #define STACK_ALIGN 16 | |
89 | ||
90 | ||
91 | #ifndef __ASSEMBLER__ | |
92 | /* Get system call information. */ | |
93 | # include <sysdep.h> | |
94 | ||
bd4f43b4 UD |
95 | #ifndef LOCK_PREFIX |
96 | # ifdef UP | |
97 | # define LOCK_PREFIX /* nothing */ | |
98 | # else | |
99 | # define LOCK_PREFIX "lock;" | |
100 | # endif | |
101 | #endif | |
102 | ||
f7d78e18 UD |
103 | /* This is the size of the initial TCB. Can't be just sizeof (tcbhead_t), |
104 | because NPTL getpid, __libc_alloca_cutoff etc. need (almost) the whole | |
105 | struct pthread even when not linked with -lpthread. */ | |
106 | # define TLS_INIT_TCB_SIZE sizeof (struct pthread) | |
a3931336 UD |
107 | |
108 | /* Alignment requirements for the initial TCB. */ | |
f7d78e18 | 109 | # define TLS_INIT_TCB_ALIGN __alignof__ (struct pthread) |
a3931336 UD |
110 | |
111 | /* This is the size of the TCB. */ | |
112 | # define TLS_TCB_SIZE sizeof (struct pthread) | |
113 | ||
114 | /* Alignment requirements for the TCB. */ | |
f9068148 | 115 | # define TLS_TCB_ALIGN __alignof__ (struct pthread) |
a3931336 UD |
116 | |
117 | /* The TCB can have any size and the memory following the address the | |
118 | thread pointer points to is unspecified. Allocate the TCB there. */ | |
119 | # define TLS_TCB_AT_TP 1 | |
498a2233 RM |
120 | # define TLS_DTV_AT_TP 0 |
121 | ||
122 | /* Get the thread descriptor definition. */ | |
123 | # include <nptl/descr.h> | |
a3931336 UD |
124 | |
125 | ||
126 | /* Install the dtv pointer. The pointer passed is to the element with | |
127 | index -1 which contain the length. */ | |
128 | # define INSTALL_DTV(descr, dtvp) \ | |
d4f64e1a | 129 | ((tcbhead_t *) (descr))->dtv = (dtvp) + 1 |
a3931336 UD |
130 | |
131 | /* Install new dtv for current thread. */ | |
d4f64e1a | 132 | # define INSTALL_NEW_DTV(dtvp) \ |
a3931336 | 133 | ({ struct pthread *__pd; \ |
55c11fbd | 134 | THREAD_SETMEM (__pd, header.dtv, (dtvp)); }) |
a3931336 UD |
135 | |
136 | /* Return dtv of given thread descriptor. */ | |
137 | # define GET_DTV(descr) \ | |
138 | (((tcbhead_t *) (descr))->dtv) | |
139 | ||
140 | ||
a3931336 UD |
141 | /* Code to initially initialize the thread pointer. This might need |
142 | special attention since 'errno' is not yet available and if the | |
143 | operation can cause a failure 'errno' must not be touched. | |
144 | ||
145 | We have to make the syscall for both uses of the macro since the | |
146 | address might be (and probably is) different. */ | |
774f9285 | 147 | # define TLS_INIT_TP(thrdescr) \ |
a3931336 UD |
148 | ({ void *_thrdescr = (thrdescr); \ |
149 | tcbhead_t *_head = _thrdescr; \ | |
150 | int _result; \ | |
151 | \ | |
152 | _head->tcb = _thrdescr; \ | |
153 | /* For now the thread descriptor is at the same address. */ \ | |
154 | _head->self = _thrdescr; \ | |
155 | \ | |
156 | /* It is a simple syscall to set the %fs value for the thread. */ \ | |
157 | asm volatile ("syscall" \ | |
158 | : "=a" (_result) \ | |
159 | : "0" ((unsigned long int) __NR_arch_prctl), \ | |
160 | "D" ((unsigned long int) ARCH_SET_FS), \ | |
3e976b96 | 161 | "S" (_thrdescr) \ |
a3931336 UD |
162 | : "memory", "cc", "r11", "cx"); \ |
163 | \ | |
fde89ad0 RM |
164 | _result ? "cannot set %fs base address for thread-local storage" : 0; \ |
165 | }) | |
a3931336 | 166 | |
962b744d RM |
167 | # define TLS_DEFINE_INIT_TP(tp, pd) void *tp = (pd) |
168 | ||
a3931336 UD |
169 | |
170 | /* Return the address of the dtv for the current thread. */ | |
171 | # define THREAD_DTV() \ | |
172 | ({ struct pthread *__pd; \ | |
55c11fbd | 173 | THREAD_GETMEM (__pd, header.dtv); }) |
a3931336 UD |
174 | |
175 | ||
176 | /* Return the thread descriptor for the current thread. | |
177 | ||
178 | The contained asm must *not* be marked volatile since otherwise | |
179 | assignments like | |
84088310 | 180 | pthread_descr self = thread_self(); |
a3931336 UD |
181 | do not get optimized away. */ |
182 | # define THREAD_SELF \ | |
183 | ({ struct pthread *__self; \ | |
512ec530 | 184 | asm ("mov %%fs:%c1,%0" : "=r" (__self) \ |
55c11fbd | 185 | : "i" (offsetof (struct pthread, header.self))); \ |
a3931336 UD |
186 | __self;}) |
187 | ||
7f08f55a RM |
188 | /* Magic for libthread_db to know how to do THREAD_SELF. */ |
189 | # define DB_THREAD_SELF_INCLUDE <sys/reg.h> /* For the FS constant. */ | |
190 | # define DB_THREAD_SELF CONST_THREAD_AREA (64, FS) | |
a3931336 UD |
191 | |
192 | /* Read member of the thread descriptor directly. */ | |
193 | # define THREAD_GETMEM(descr, member) \ | |
194 | ({ __typeof (descr->member) __value; \ | |
195 | if (sizeof (__value) == 1) \ | |
7f293686 UD |
196 | asm volatile ("movb %%fs:%P2,%b0" \ |
197 | : "=q" (__value) \ | |
198 | : "0" (0), "i" (offsetof (struct pthread, member))); \ | |
a3931336 | 199 | else if (sizeof (__value) == 4) \ |
7f293686 UD |
200 | asm volatile ("movl %%fs:%P1,%0" \ |
201 | : "=r" (__value) \ | |
202 | : "i" (offsetof (struct pthread, member))); \ | |
a3931336 UD |
203 | else \ |
204 | { \ | |
205 | if (sizeof (__value) != 8) \ | |
206 | /* There should not be any value with a size other than 1, \ | |
207 | 4 or 8. */ \ | |
208 | abort (); \ | |
209 | \ | |
7f293686 UD |
210 | asm volatile ("movq %%fs:%P1,%q0" \ |
211 | : "=r" (__value) \ | |
212 | : "i" (offsetof (struct pthread, member))); \ | |
a3931336 UD |
213 | } \ |
214 | __value; }) | |
215 | ||
216 | ||
217 | /* Same as THREAD_GETMEM, but the member offset can be non-constant. */ | |
218 | # define THREAD_GETMEM_NC(descr, member, idx) \ | |
219 | ({ __typeof (descr->member[0]) __value; \ | |
220 | if (sizeof (__value) == 1) \ | |
7f293686 UD |
221 | asm volatile ("movb %%fs:%P2(%q3),%b0" \ |
222 | : "=q" (__value) \ | |
223 | : "0" (0), "i" (offsetof (struct pthread, member[0])), \ | |
224 | "r" (idx)); \ | |
a3931336 | 225 | else if (sizeof (__value) == 4) \ |
7f293686 UD |
226 | asm volatile ("movl %%fs:%P1(,%q2,4),%0" \ |
227 | : "=r" (__value) \ | |
228 | : "i" (offsetof (struct pthread, member[0])), "r" (idx));\ | |
a3931336 UD |
229 | else \ |
230 | { \ | |
231 | if (sizeof (__value) != 8) \ | |
232 | /* There should not be any value with a size other than 1, \ | |
233 | 4 or 8. */ \ | |
234 | abort (); \ | |
235 | \ | |
7f293686 UD |
236 | asm volatile ("movq %%fs:%P1(,%q2,8),%q0" \ |
237 | : "=r" (__value) \ | |
238 | : "i" (offsetof (struct pthread, member[0])), \ | |
239 | "r" (idx)); \ | |
a3931336 UD |
240 | } \ |
241 | __value; }) | |
242 | ||
243 | ||
177d1ad3 UD |
244 | /* Loading addresses of objects on x86-64 needs to be treated special |
245 | when generating PIC code. */ | |
246 | #ifdef __pic__ | |
247 | # define IMM_MODE "nr" | |
248 | #else | |
249 | # define IMM_MODE "ir" | |
250 | #endif | |
251 | ||
252 | ||
a3931336 UD |
253 | /* Same as THREAD_SETMEM, but the member offset can be non-constant. */ |
254 | # define THREAD_SETMEM(descr, member, value) \ | |
255 | ({ if (sizeof (descr->member) == 1) \ | |
5a03acfe | 256 | asm volatile ("movb %b0,%%fs:%P1" : \ |
a3931336 UD |
257 | : "iq" (value), \ |
258 | "i" (offsetof (struct pthread, member))); \ | |
259 | else if (sizeof (descr->member) == 4) \ | |
260 | asm volatile ("movl %0,%%fs:%P1" : \ | |
177d1ad3 | 261 | : IMM_MODE (value), \ |
a3931336 UD |
262 | "i" (offsetof (struct pthread, member))); \ |
263 | else \ | |
264 | { \ | |
265 | if (sizeof (descr->member) != 8) \ | |
266 | /* There should not be any value with a size other than 1, \ | |
267 | 4 or 8. */ \ | |
268 | abort (); \ | |
269 | \ | |
5a03acfe | 270 | asm volatile ("movq %q0,%%fs:%P1" : \ |
c515fb51 | 271 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
a3931336 UD |
272 | "i" (offsetof (struct pthread, member))); \ |
273 | }}) | |
274 | ||
275 | ||
276 | /* Set member of the thread descriptor directly. */ | |
277 | # define THREAD_SETMEM_NC(descr, member, idx, value) \ | |
278 | ({ if (sizeof (descr->member[0]) == 1) \ | |
5a03acfe | 279 | asm volatile ("movb %b0,%%fs:%P1(%q2)" : \ |
a3931336 UD |
280 | : "iq" (value), \ |
281 | "i" (offsetof (struct pthread, member[0])), \ | |
282 | "r" (idx)); \ | |
283 | else if (sizeof (descr->member[0]) == 4) \ | |
5a03acfe | 284 | asm volatile ("movl %0,%%fs:%P1(,%q2,4)" : \ |
177d1ad3 | 285 | : IMM_MODE (value), \ |
a3931336 UD |
286 | "i" (offsetof (struct pthread, member[0])), \ |
287 | "r" (idx)); \ | |
288 | else \ | |
289 | { \ | |
290 | if (sizeof (descr->member[0]) != 8) \ | |
291 | /* There should not be any value with a size other than 1, \ | |
292 | 4 or 8. */ \ | |
293 | abort (); \ | |
294 | \ | |
5a03acfe | 295 | asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" : \ |
c515fb51 | 296 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
a3931336 UD |
297 | "i" (offsetof (struct pthread, member[0])), \ |
298 | "r" (idx)); \ | |
299 | }}) | |
300 | ||
301 | ||
b22d701b | 302 | /* Atomic compare and exchange on TLS, returning old value. */ |
b48a267b | 303 | # define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \ |
b22d701b UD |
304 | ({ __typeof (descr->member) __ret; \ |
305 | __typeof (oldval) __old = (oldval); \ | |
306 | if (sizeof (descr->member) == 4) \ | |
bd4f43b4 | 307 | asm volatile (LOCK_PREFIX "cmpxchgl %2, %%fs:%P3" \ |
b22d701b UD |
308 | : "=a" (__ret) \ |
309 | : "0" (__old), "r" (newval), \ | |
310 | "i" (offsetof (struct pthread, member))); \ | |
311 | else \ | |
312 | /* Not necessary for other sizes in the moment. */ \ | |
313 | abort (); \ | |
314 | __ret; }) | |
315 | ||
316 | ||
d5c157a9 | 317 | /* Atomic logical and. */ |
b48a267b | 318 | # define THREAD_ATOMIC_AND(descr, member, val) \ |
d5c157a9 UD |
319 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
320 | asm volatile (LOCK_PREFIX "andl %1, %%fs:%P0" \ | |
321 | :: "i" (offsetof (struct pthread, member)), \ | |
322 | "ir" (val)); \ | |
323 | else \ | |
324 | /* Not necessary for other sizes in the moment. */ \ | |
325 | abort (); }) | |
326 | ||
327 | ||
b22d701b | 328 | /* Atomic set bit. */ |
b48a267b | 329 | # define THREAD_ATOMIC_BIT_SET(descr, member, bit) \ |
b22d701b | 330 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
bd4f43b4 | 331 | asm volatile (LOCK_PREFIX "orl %1, %%fs:%P0" \ |
b22d701b UD |
332 | :: "i" (offsetof (struct pthread, member)), \ |
333 | "ir" (1 << (bit))); \ | |
334 | else \ | |
335 | /* Not necessary for other sizes in the moment. */ \ | |
336 | abort (); }) | |
337 | ||
338 | ||
b48a267b | 339 | # define CALL_THREAD_FCT(descr) \ |
a87731e2 UD |
340 | ({ void *__res; \ |
341 | asm volatile ("movq %%fs:%P2, %%rdi\n\t" \ | |
342 | "callq *%%fs:%P1" \ | |
343 | : "=a" (__res) \ | |
344 | : "i" (offsetof (struct pthread, start_routine)), \ | |
345 | "i" (offsetof (struct pthread, arg)) \ | |
346 | : "di", "si", "cx", "dx", "r8", "r9", "r10", "r11", \ | |
347 | "memory", "cc"); \ | |
348 | __res; }) | |
349 | ||
350 | ||
35f1e827 UD |
351 | /* Set the stack guard field in TCB head. */ |
352 | # define THREAD_SET_STACK_GUARD(value) \ | |
353 | THREAD_SETMEM (THREAD_SELF, header.stack_guard, value) | |
354 | # define THREAD_COPY_STACK_GUARD(descr) \ | |
355 | ((descr)->header.stack_guard \ | |
356 | = THREAD_GETMEM (THREAD_SELF, header.stack_guard)) | |
357 | ||
827b7087 UD |
358 | |
359 | /* Set the pointer guard field in the TCB head. */ | |
b48a267b | 360 | # define THREAD_SET_POINTER_GUARD(value) \ |
827b7087 | 361 | THREAD_SETMEM (THREAD_SELF, header.pointer_guard, value) |
b48a267b | 362 | # define THREAD_COPY_POINTER_GUARD(descr) \ |
827b7087 UD |
363 | ((descr)->header.pointer_guard \ |
364 | = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) | |
365 | ||
366 | ||
df94b641 | 367 | /* Get and set the global scope generation counter in the TCB head. */ |
b48a267b UD |
368 | # define THREAD_GSCOPE_FLAG_UNUSED 0 |
369 | # define THREAD_GSCOPE_FLAG_USED 1 | |
370 | # define THREAD_GSCOPE_FLAG_WAIT 2 | |
371 | # define THREAD_GSCOPE_RESET_FLAG() \ | |
df94b641 UD |
372 | do \ |
373 | { int __res; \ | |
374 | asm volatile ("xchgl %0, %%fs:%P1" \ | |
375 | : "=r" (__res) \ | |
376 | : "i" (offsetof (struct pthread, header.gscope_flag)), \ | |
377 | "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ | |
378 | if (__res == THREAD_GSCOPE_FLAG_WAIT) \ | |
085a4412 | 379 | lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1, LLL_PRIVATE); \ |
df94b641 UD |
380 | } \ |
381 | while (0) | |
b48a267b | 382 | # define THREAD_GSCOPE_SET_FLAG() \ |
df94b641 | 383 | THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) |
b48a267b | 384 | # define THREAD_GSCOPE_WAIT() \ |
6d59823c | 385 | GL(dl_wait_lookup_done) () |
df94b641 | 386 | |
b48a267b UD |
387 | |
388 | # ifdef SHARED | |
389 | /* Defined in dl-trampoline.S. */ | |
390 | extern void _dl_x86_64_save_sse (void); | |
391 | extern void _dl_x86_64_restore_sse (void); | |
392 | ||
393 | # define RTLD_CHECK_FOREIGN_CALL \ | |
394 | (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save) != 0) | |
395 | ||
84088310 UD |
396 | /* NB: Don't use the xchg operation because that would imply a lock |
397 | prefix which is expensive and unnecessary. The cache line is also | |
398 | not contested at all. */ | |
b48a267b | 399 | # define RTLD_ENABLE_FOREIGN_CALL \ |
84088310 UD |
400 | int old_rtld_must_xmm_save = THREAD_GETMEM (THREAD_SELF, \ |
401 | header.rtld_must_xmm_save); \ | |
b48a267b UD |
402 | THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, 1) |
403 | ||
404 | # define RTLD_PREPARE_FOREIGN_CALL \ | |
405 | do if (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save)) \ | |
406 | { \ | |
407 | _dl_x86_64_save_sse (); \ | |
408 | THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, 0); \ | |
409 | } \ | |
410 | while (0) | |
411 | ||
412 | # define RTLD_FINALIZE_FOREIGN_CALL \ | |
413 | do { \ | |
414 | if (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save) == 0) \ | |
415 | _dl_x86_64_restore_sse (); \ | |
84088310 UD |
416 | THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, \ |
417 | old_rtld_must_xmm_save); \ | |
b48a267b UD |
418 | } while (0) |
419 | # endif | |
420 | ||
421 | ||
a3931336 UD |
422 | #endif /* __ASSEMBLER__ */ |
423 | ||
424 | #endif /* tls.h */ |