]> git.ipfire.org Git - thirdparty/glibc.git/blob - nptl/pthread_create.c
Use glibc_likely instead __builtin_expect.
[thirdparty/glibc.git] / nptl / pthread_create.c
1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19 #include <ctype.h>
20 #include <errno.h>
21 #include <stdbool.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <stdint.h>
25 #include "pthreadP.h"
26 #include <hp-timing.h>
27 #include <ldsodefs.h>
28 #include <atomic.h>
29 #include <libc-internal.h>
30 #include <resolv.h>
31 #include <kernel-features.h>
32
33 #include <shlib-compat.h>
34
35 #include <stap-probe.h>
36
37
38 /* Local function to start thread and handle cleanup. */
39 static int start_thread (void *arg);
40
41
42 /* Nozero if debugging mode is enabled. */
43 int __pthread_debug;
44
45 /* Globally enabled events. */
46 static td_thr_events_t __nptl_threads_events __attribute_used__;
47
48 /* Pointer to descriptor with the last event. */
49 static struct pthread *__nptl_last_event __attribute_used__;
50
51 /* Number of threads running. */
52 unsigned int __nptl_nthreads = 1;
53
54
55 /* Code to allocate and deallocate a stack. */
56 #include "allocatestack.c"
57
58 /* Code to create the thread. */
59 #include <createthread.c>
60
61
62 struct pthread *
63 internal_function
64 __find_in_stack_list (pd)
65 struct pthread *pd;
66 {
67 list_t *entry;
68 struct pthread *result = NULL;
69
70 lll_lock (stack_cache_lock, LLL_PRIVATE);
71
72 list_for_each (entry, &stack_used)
73 {
74 struct pthread *curp;
75
76 curp = list_entry (entry, struct pthread, list);
77 if (curp == pd)
78 {
79 result = curp;
80 break;
81 }
82 }
83
84 if (result == NULL)
85 list_for_each (entry, &__stack_user)
86 {
87 struct pthread *curp;
88
89 curp = list_entry (entry, struct pthread, list);
90 if (curp == pd)
91 {
92 result = curp;
93 break;
94 }
95 }
96
97 lll_unlock (stack_cache_lock, LLL_PRIVATE);
98
99 return result;
100 }
101
102
103 /* Deallocate POSIX thread-local-storage. */
104 void
105 attribute_hidden
106 __nptl_deallocate_tsd (void)
107 {
108 struct pthread *self = THREAD_SELF;
109
110 /* Maybe no data was ever allocated. This happens often so we have
111 a flag for this. */
112 if (THREAD_GETMEM (self, specific_used))
113 {
114 size_t round;
115 size_t cnt;
116
117 round = 0;
118 do
119 {
120 size_t idx;
121
122 /* So far no new nonzero data entry. */
123 THREAD_SETMEM (self, specific_used, false);
124
125 for (cnt = idx = 0; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
126 {
127 struct pthread_key_data *level2;
128
129 level2 = THREAD_GETMEM_NC (self, specific, cnt);
130
131 if (level2 != NULL)
132 {
133 size_t inner;
134
135 for (inner = 0; inner < PTHREAD_KEY_2NDLEVEL_SIZE;
136 ++inner, ++idx)
137 {
138 void *data = level2[inner].data;
139
140 if (data != NULL)
141 {
142 /* Always clear the data. */
143 level2[inner].data = NULL;
144
145 /* Make sure the data corresponds to a valid
146 key. This test fails if the key was
147 deallocated and also if it was
148 re-allocated. It is the user's
149 responsibility to free the memory in this
150 case. */
151 if (level2[inner].seq
152 == __pthread_keys[idx].seq
153 /* It is not necessary to register a destructor
154 function. */
155 && __pthread_keys[idx].destr != NULL)
156 /* Call the user-provided destructor. */
157 __pthread_keys[idx].destr (data);
158 }
159 }
160 }
161 else
162 idx += PTHREAD_KEY_1STLEVEL_SIZE;
163 }
164
165 if (THREAD_GETMEM (self, specific_used) == 0)
166 /* No data has been modified. */
167 goto just_free;
168 }
169 /* We only repeat the process a fixed number of times. */
170 while (__builtin_expect (++round < PTHREAD_DESTRUCTOR_ITERATIONS, 0));
171
172 /* Just clear the memory of the first block for reuse. */
173 memset (&THREAD_SELF->specific_1stblock, '\0',
174 sizeof (self->specific_1stblock));
175
176 just_free:
177 /* Free the memory for the other blocks. */
178 for (cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
179 {
180 struct pthread_key_data *level2;
181
182 level2 = THREAD_GETMEM_NC (self, specific, cnt);
183 if (level2 != NULL)
184 {
185 /* The first block is allocated as part of the thread
186 descriptor. */
187 free (level2);
188 THREAD_SETMEM_NC (self, specific, cnt, NULL);
189 }
190 }
191
192 THREAD_SETMEM (self, specific_used, false);
193 }
194 }
195
196
197 /* Deallocate a thread's stack after optionally making sure the thread
198 descriptor is still valid. */
199 void
200 internal_function
201 __free_tcb (struct pthread *pd)
202 {
203 /* The thread is exiting now. */
204 if (__builtin_expect (atomic_bit_test_set (&pd->cancelhandling,
205 TERMINATED_BIT) == 0, 1))
206 {
207 /* Remove the descriptor from the list. */
208 if (DEBUGGING_P && __find_in_stack_list (pd) == NULL)
209 /* Something is really wrong. The descriptor for a still
210 running thread is gone. */
211 abort ();
212
213 /* Free TPP data. */
214 if (__glibc_unlikely (pd->tpp != NULL))
215 {
216 struct priority_protection_data *tpp = pd->tpp;
217
218 pd->tpp = NULL;
219 free (tpp);
220 }
221
222 /* Queue the stack memory block for reuse and exit the process. The
223 kernel will signal via writing to the address returned by
224 QUEUE-STACK when the stack is available. */
225 __deallocate_stack (pd);
226 }
227 }
228
229
230 static int
231 start_thread (void *arg)
232 {
233 struct pthread *pd = (struct pthread *) arg;
234
235 #if HP_TIMING_AVAIL
236 /* Remember the time when the thread was started. */
237 hp_timing_t now;
238 HP_TIMING_NOW (now);
239 THREAD_SETMEM (pd, cpuclock_offset, now);
240 #endif
241
242 /* Initialize resolver state pointer. */
243 __resp = &pd->res;
244
245 /* Initialize pointers to locale data. */
246 __ctype_init ();
247
248 /* Allow setxid from now onwards. */
249 if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2))
250 lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
251
252 #ifdef __NR_set_robust_list
253 # ifndef __ASSUME_SET_ROBUST_LIST
254 if (__set_robust_list_avail >= 0)
255 # endif
256 {
257 INTERNAL_SYSCALL_DECL (err);
258 /* This call should never fail because the initial call in init.c
259 succeeded. */
260 INTERNAL_SYSCALL (set_robust_list, err, 2, &pd->robust_head,
261 sizeof (struct robust_list_head));
262 }
263 #endif
264
265 /* If the parent was running cancellation handlers while creating
266 the thread the new thread inherited the signal mask. Reset the
267 cancellation signal mask. */
268 if (__glibc_unlikely (pd->parent_cancelhandling & CANCELING_BITMASK))
269 {
270 INTERNAL_SYSCALL_DECL (err);
271 sigset_t mask;
272 __sigemptyset (&mask);
273 __sigaddset (&mask, SIGCANCEL);
274 (void) INTERNAL_SYSCALL (rt_sigprocmask, err, 4, SIG_UNBLOCK, &mask,
275 NULL, _NSIG / 8);
276 }
277
278 /* This is where the try/finally block should be created. For
279 compilers without that support we do use setjmp. */
280 struct pthread_unwind_buf unwind_buf;
281
282 /* No previous handlers. */
283 unwind_buf.priv.data.prev = NULL;
284 unwind_buf.priv.data.cleanup = NULL;
285
286 int not_first_call;
287 not_first_call = setjmp ((struct __jmp_buf_tag *) unwind_buf.cancel_jmp_buf);
288 if (__glibc_likely (! not_first_call))
289 {
290 /* Store the new cleanup handler info. */
291 THREAD_SETMEM (pd, cleanup_jmp_buf, &unwind_buf);
292
293 if (__glibc_unlikely (pd->stopped_start))
294 {
295 int oldtype = CANCEL_ASYNC ();
296
297 /* Get the lock the parent locked to force synchronization. */
298 lll_lock (pd->lock, LLL_PRIVATE);
299 /* And give it up right away. */
300 lll_unlock (pd->lock, LLL_PRIVATE);
301
302 CANCEL_RESET (oldtype);
303 }
304
305 LIBC_PROBE (pthread_start, 3, (pthread_t) pd, pd->start_routine, pd->arg);
306
307 /* Run the code the user provided. */
308 #ifdef CALL_THREAD_FCT
309 THREAD_SETMEM (pd, result, CALL_THREAD_FCT (pd));
310 #else
311 THREAD_SETMEM (pd, result, pd->start_routine (pd->arg));
312 #endif
313 }
314
315 /* Call destructors for the thread_local TLS variables. */
316 #ifndef SHARED
317 if (&__call_tls_dtors != NULL)
318 #endif
319 __call_tls_dtors ();
320
321 /* Run the destructor for the thread-local data. */
322 __nptl_deallocate_tsd ();
323
324 /* Clean up any state libc stored in thread-local variables. */
325 __libc_thread_freeres ();
326
327 /* If this is the last thread we terminate the process now. We
328 do not notify the debugger, it might just irritate it if there
329 is no thread left. */
330 if (__glibc_unlikely (atomic_decrement_and_test (&__nptl_nthreads)))
331 /* This was the last thread. */
332 exit (0);
333
334 /* Report the death of the thread if this is wanted. */
335 if (__glibc_unlikely (pd->report_events))
336 {
337 /* See whether TD_DEATH is in any of the mask. */
338 const int idx = __td_eventword (TD_DEATH);
339 const uint32_t mask = __td_eventmask (TD_DEATH);
340
341 if ((mask & (__nptl_threads_events.event_bits[idx]
342 | pd->eventbuf.eventmask.event_bits[idx])) != 0)
343 {
344 /* Yep, we have to signal the death. Add the descriptor to
345 the list but only if it is not already on it. */
346 if (pd->nextevent == NULL)
347 {
348 pd->eventbuf.eventnum = TD_DEATH;
349 pd->eventbuf.eventdata = pd;
350
351 do
352 pd->nextevent = __nptl_last_event;
353 while (atomic_compare_and_exchange_bool_acq (&__nptl_last_event,
354 pd, pd->nextevent));
355 }
356
357 /* Now call the function to signal the event. */
358 __nptl_death_event ();
359 }
360 }
361
362 /* The thread is exiting now. Don't set this bit until after we've hit
363 the event-reporting breakpoint, so that td_thr_get_info on us while at
364 the breakpoint reports TD_THR_RUN state rather than TD_THR_ZOMBIE. */
365 atomic_bit_set (&pd->cancelhandling, EXITING_BIT);
366
367 #ifndef __ASSUME_SET_ROBUST_LIST
368 /* If this thread has any robust mutexes locked, handle them now. */
369 # ifdef __PTHREAD_MUTEX_HAVE_PREV
370 void *robust = pd->robust_head.list;
371 # else
372 __pthread_slist_t *robust = pd->robust_list.__next;
373 # endif
374 /* We let the kernel do the notification if it is able to do so.
375 If we have to do it here there for sure are no PI mutexes involved
376 since the kernel support for them is even more recent. */
377 if (__set_robust_list_avail < 0
378 && __builtin_expect (robust != (void *) &pd->robust_head, 0))
379 {
380 do
381 {
382 struct __pthread_mutex_s *this = (struct __pthread_mutex_s *)
383 ((char *) robust - offsetof (struct __pthread_mutex_s,
384 __list.__next));
385 robust = *((void **) robust);
386
387 # ifdef __PTHREAD_MUTEX_HAVE_PREV
388 this->__list.__prev = NULL;
389 # endif
390 this->__list.__next = NULL;
391
392 lll_robust_dead (this->__lock, /* XYZ */ LLL_SHARED);
393 }
394 while (robust != (void *) &pd->robust_head);
395 }
396 #endif
397
398 /* Mark the memory of the stack as usable to the kernel. We free
399 everything except for the space used for the TCB itself. */
400 size_t pagesize_m1 = __getpagesize () - 1;
401 #ifdef _STACK_GROWS_DOWN
402 char *sp = CURRENT_STACK_FRAME;
403 size_t freesize = (sp - (char *) pd->stackblock) & ~pagesize_m1;
404 #else
405 # error "to do"
406 #endif
407 assert (freesize < pd->stackblock_size);
408 if (freesize > PTHREAD_STACK_MIN)
409 __madvise (pd->stackblock, freesize - PTHREAD_STACK_MIN, MADV_DONTNEED);
410
411 /* If the thread is detached free the TCB. */
412 if (IS_DETACHED (pd))
413 /* Free the TCB. */
414 __free_tcb (pd);
415 else if (__glibc_unlikely (pd->cancelhandling & SETXID_BITMASK))
416 {
417 /* Some other thread might call any of the setXid functions and expect
418 us to reply. In this case wait until we did that. */
419 do
420 lll_futex_wait (&pd->setxid_futex, 0, LLL_PRIVATE);
421 while (pd->cancelhandling & SETXID_BITMASK);
422
423 /* Reset the value so that the stack can be reused. */
424 pd->setxid_futex = 0;
425 }
426
427 /* We cannot call '_exit' here. '_exit' will terminate the process.
428
429 The 'exit' implementation in the kernel will signal when the
430 process is really dead since 'clone' got passed the CLONE_CHILD_CLEARTID
431 flag. The 'tid' field in the TCB will be set to zero.
432
433 The exit code is zero since in case all threads exit by calling
434 'pthread_exit' the exit status must be 0 (zero). */
435 __exit_thread_inline (0);
436
437 /* NOTREACHED */
438 return 0;
439 }
440
441
442 int
443 __pthread_create_2_1 (newthread, attr, start_routine, arg)
444 pthread_t *newthread;
445 const pthread_attr_t *attr;
446 void *(*start_routine) (void *);
447 void *arg;
448 {
449 STACK_VARIABLES;
450
451 const struct pthread_attr *iattr = (struct pthread_attr *) attr;
452 struct pthread_attr default_attr;
453 bool free_cpuset = false;
454 if (iattr == NULL)
455 {
456 lll_lock (__default_pthread_attr_lock, LLL_PRIVATE);
457 default_attr = __default_pthread_attr;
458 size_t cpusetsize = default_attr.cpusetsize;
459 if (cpusetsize > 0)
460 {
461 cpu_set_t *cpuset;
462 if (__glibc_likely (__libc_use_alloca (cpusetsize)))
463 cpuset = __alloca (cpusetsize);
464 else
465 {
466 cpuset = malloc (cpusetsize);
467 if (cpuset == NULL)
468 {
469 lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
470 return ENOMEM;
471 }
472 free_cpuset = true;
473 }
474 memcpy (cpuset, default_attr.cpuset, cpusetsize);
475 default_attr.cpuset = cpuset;
476 }
477 lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
478 iattr = &default_attr;
479 }
480
481 struct pthread *pd = NULL;
482 int err = ALLOCATE_STACK (iattr, &pd);
483 int retval = 0;
484
485 if (__glibc_unlikely (err != 0))
486 /* Something went wrong. Maybe a parameter of the attributes is
487 invalid or we could not allocate memory. Note we have to
488 translate error codes. */
489 {
490 retval = err == ENOMEM ? EAGAIN : err;
491 goto out;
492 }
493
494
495 /* Initialize the TCB. All initializations with zero should be
496 performed in 'get_cached_stack'. This way we avoid doing this if
497 the stack freshly allocated with 'mmap'. */
498
499 #ifdef TLS_TCB_AT_TP
500 /* Reference to the TCB itself. */
501 pd->header.self = pd;
502
503 /* Self-reference for TLS. */
504 pd->header.tcb = pd;
505 #endif
506
507 /* Store the address of the start routine and the parameter. Since
508 we do not start the function directly the stillborn thread will
509 get the information from its thread descriptor. */
510 pd->start_routine = start_routine;
511 pd->arg = arg;
512
513 /* Copy the thread attribute flags. */
514 struct pthread *self = THREAD_SELF;
515 pd->flags = ((iattr->flags & ~(ATTR_FLAG_SCHED_SET | ATTR_FLAG_POLICY_SET))
516 | (self->flags & (ATTR_FLAG_SCHED_SET | ATTR_FLAG_POLICY_SET)));
517
518 /* Initialize the field for the ID of the thread which is waiting
519 for us. This is a self-reference in case the thread is created
520 detached. */
521 pd->joinid = iattr->flags & ATTR_FLAG_DETACHSTATE ? pd : NULL;
522
523 /* The debug events are inherited from the parent. */
524 pd->eventbuf = self->eventbuf;
525
526
527 /* Copy the parent's scheduling parameters. The flags will say what
528 is valid and what is not. */
529 pd->schedpolicy = self->schedpolicy;
530 pd->schedparam = self->schedparam;
531
532 /* Copy the stack guard canary. */
533 #ifdef THREAD_COPY_STACK_GUARD
534 THREAD_COPY_STACK_GUARD (pd);
535 #endif
536
537 /* Copy the pointer guard value. */
538 #ifdef THREAD_COPY_POINTER_GUARD
539 THREAD_COPY_POINTER_GUARD (pd);
540 #endif
541
542 /* Determine scheduling parameters for the thread. */
543 if (__builtin_expect ((iattr->flags & ATTR_FLAG_NOTINHERITSCHED) != 0, 0)
544 && (iattr->flags & (ATTR_FLAG_SCHED_SET | ATTR_FLAG_POLICY_SET)) != 0)
545 {
546 INTERNAL_SYSCALL_DECL (scerr);
547
548 /* Use the scheduling parameters the user provided. */
549 if (iattr->flags & ATTR_FLAG_POLICY_SET)
550 pd->schedpolicy = iattr->schedpolicy;
551 else if ((pd->flags & ATTR_FLAG_POLICY_SET) == 0)
552 {
553 pd->schedpolicy = INTERNAL_SYSCALL (sched_getscheduler, scerr, 1, 0);
554 pd->flags |= ATTR_FLAG_POLICY_SET;
555 }
556
557 if (iattr->flags & ATTR_FLAG_SCHED_SET)
558 memcpy (&pd->schedparam, &iattr->schedparam,
559 sizeof (struct sched_param));
560 else if ((pd->flags & ATTR_FLAG_SCHED_SET) == 0)
561 {
562 INTERNAL_SYSCALL (sched_getparam, scerr, 2, 0, &pd->schedparam);
563 pd->flags |= ATTR_FLAG_SCHED_SET;
564 }
565
566 /* Check for valid priorities. */
567 int minprio = INTERNAL_SYSCALL (sched_get_priority_min, scerr, 1,
568 iattr->schedpolicy);
569 int maxprio = INTERNAL_SYSCALL (sched_get_priority_max, scerr, 1,
570 iattr->schedpolicy);
571 if (pd->schedparam.sched_priority < minprio
572 || pd->schedparam.sched_priority > maxprio)
573 {
574 /* Perhaps a thread wants to change the IDs and if waiting
575 for this stillborn thread. */
576 if (__builtin_expect (atomic_exchange_acq (&pd->setxid_futex, 0)
577 == -2, 0))
578 lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
579
580 __deallocate_stack (pd);
581
582 retval = EINVAL;
583 goto out;
584 }
585 }
586
587 /* Pass the descriptor to the caller. */
588 *newthread = (pthread_t) pd;
589
590 LIBC_PROBE (pthread_create, 4, newthread, attr, start_routine, arg);
591
592 /* Start the thread. */
593 retval = create_thread (pd, iattr, STACK_VARIABLES_ARGS);
594
595 out:
596 if (__glibc_unlikely (free_cpuset))
597 free (default_attr.cpuset);
598
599 return retval;
600 }
601 versioned_symbol (libpthread, __pthread_create_2_1, pthread_create, GLIBC_2_1);
602
603
604 #if SHLIB_COMPAT(libpthread, GLIBC_2_0, GLIBC_2_1)
605 int
606 __pthread_create_2_0 (newthread, attr, start_routine, arg)
607 pthread_t *newthread;
608 const pthread_attr_t *attr;
609 void *(*start_routine) (void *);
610 void *arg;
611 {
612 /* The ATTR attribute is not really of type `pthread_attr_t *'. It has
613 the old size and access to the new members might crash the program.
614 We convert the struct now. */
615 struct pthread_attr new_attr;
616
617 if (attr != NULL)
618 {
619 struct pthread_attr *iattr = (struct pthread_attr *) attr;
620 size_t ps = __getpagesize ();
621
622 /* Copy values from the user-provided attributes. */
623 new_attr.schedparam = iattr->schedparam;
624 new_attr.schedpolicy = iattr->schedpolicy;
625 new_attr.flags = iattr->flags;
626
627 /* Fill in default values for the fields not present in the old
628 implementation. */
629 new_attr.guardsize = ps;
630 new_attr.stackaddr = NULL;
631 new_attr.stacksize = 0;
632 new_attr.cpuset = NULL;
633
634 /* We will pass this value on to the real implementation. */
635 attr = (pthread_attr_t *) &new_attr;
636 }
637
638 return __pthread_create_2_1 (newthread, attr, start_routine, arg);
639 }
640 compat_symbol (libpthread, __pthread_create_2_0, pthread_create,
641 GLIBC_2_0);
642 #endif
643 \f
644 /* Information for libthread_db. */
645
646 #include "../nptl_db/db_info.c"
647 \f
648 /* If pthread_create is present, libgcc_eh.a and libsupc++.a expects some other POSIX thread
649 functions to be present as well. */
650 PTHREAD_STATIC_FN_REQUIRE (pthread_mutex_lock)
651 PTHREAD_STATIC_FN_REQUIRE (pthread_mutex_trylock)
652 PTHREAD_STATIC_FN_REQUIRE (pthread_mutex_unlock)
653
654 PTHREAD_STATIC_FN_REQUIRE (pthread_once)
655 PTHREAD_STATIC_FN_REQUIRE (pthread_cancel)
656
657 PTHREAD_STATIC_FN_REQUIRE (pthread_key_create)
658 PTHREAD_STATIC_FN_REQUIRE (pthread_key_delete)
659 PTHREAD_STATIC_FN_REQUIRE (pthread_setspecific)
660 PTHREAD_STATIC_FN_REQUIRE (pthread_getspecific)