-/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
list_t __stack_user __attribute__ ((nocommon));
hidden_data_def (__stack_user)
-#if COLORING_INCREMENT != 0
-/* Number of threads created. */
-static unsigned int nptl_ncreated;
-#endif
-
/* Check whether the stack is still used or not. */
#define FREE_P(descr) ((descr)->tid <= 0)
/* Free stacks until cache size is lower than LIMIT. */
-void
-__free_stacks (size_t limit)
+static void
+free_stacks (size_t limit)
{
/* We reduce the size of the cache. Remove the last entries until
the size is below the limit. */
/* Remove this block. This should never fail. If it does
something is really wrong. */
- if (munmap (curr->stackblock, curr->stackblock_size) != 0)
+ if (__munmap (curr->stackblock, curr->stackblock_size) != 0)
abort ();
/* Maybe we have freed enough. */
}
}
+/* Free all the stacks on cleanup. */
+void
+__nptl_stacks_freeres (void)
+{
+ free_stacks (0);
+}
/* Add a stack frame which is not used anymore to the stack. Must be
called with the cache lock held. */
stack_cache_actsize += stack->stackblock_size;
if (__glibc_unlikely (stack_cache_actsize > stack_cache_maxsize))
- __free_stacks (stack_cache_maxsize);
+ free_stacks (stack_cache_maxsize);
}
static int
-internal_function
change_stack_perm (struct pthread *pd
#ifdef NEED_SEPARATE_REGISTER_STACK
, size_t pagemask
#else
# error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
#endif
- if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
+ if (__mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
return errno;
return 0;
}
+/* Return the guard page position on allocated stack. */
+static inline char *
+__attribute ((always_inline))
+guard_position (void *mem, size_t size, size_t guardsize, struct pthread *pd,
+ size_t pagesize_m1)
+{
+#ifdef NEED_SEPARATE_REGISTER_STACK
+ return mem + (((size - guardsize) / 2) & ~pagesize_m1);
+#elif _STACK_GROWS_DOWN
+ return mem;
+#elif _STACK_GROWS_UP
+ return (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
+#endif
+}
+
+/* Based on stack allocated with PROT_NONE, setup the required portions with
+ 'prot' flags based on the guard page position. */
+static inline int
+setup_stack_prot (char *mem, size_t size, char *guard, size_t guardsize,
+ const int prot)
+{
+ char *guardend = guard + guardsize;
+#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
+ /* As defined at guard_position, for architectures with downward stack
+ the guard page is always at start of the allocated area. */
+ if (__mprotect (guardend, size - guardsize, prot) != 0)
+ return errno;
+#else
+ size_t mprots1 = (uintptr_t) guard - (uintptr_t) mem;
+ if (__mprotect (mem, mprots1, prot) != 0)
+ return errno;
+ size_t mprots2 = ((uintptr_t) mem + size) - (uintptr_t) guardend;
+ if (__mprotect (guardend, mprots2, prot) != 0)
+ return errno;
+#endif
+ return 0;
+}
+
+/* Mark the memory of the stack as usable to the kernel. It frees everything
+ except for the space used for the TCB itself. */
+static inline void
+__always_inline
+advise_stack_range (void *mem, size_t size, uintptr_t pd, size_t guardsize)
+{
+ uintptr_t sp = (uintptr_t) CURRENT_STACK_FRAME;
+ size_t pagesize_m1 = __getpagesize () - 1;
+#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
+ size_t freesize = (sp - (uintptr_t) mem) & ~pagesize_m1;
+ assert (freesize < size);
+ if (freesize > PTHREAD_STACK_MIN)
+ __madvise (mem, freesize - PTHREAD_STACK_MIN, MADV_DONTNEED);
+#else
+ /* Page aligned start of memory to free (higher than or equal
+ to current sp plus the minimum stack size). */
+ uintptr_t freeblock = (sp + PTHREAD_STACK_MIN + pagesize_m1) & ~pagesize_m1;
+ uintptr_t free_end = (pd - guardsize) & ~pagesize_m1;
+ if (free_end > freeblock)
+ {
+ size_t freesize = free_end - freeblock;
+ assert (freesize < size);
+ __madvise ((void*) freeblock, freesize, MADV_DONTNEED);
+ }
+#endif
+}
/* Returns a usable stack for a new thread either by allocating a
new stack or reusing a cached stack of sufficient size.
__pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
#endif
-#ifndef __ASSUME_PRIVATE_FUTEX
- /* The thread must know when private futexes are supported. */
- pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
- header.private_futex);
-#endif
-
#ifdef NEED_DL_SYSINFO
SETUP_THREAD_SYSINFO (pd);
#endif
const int prot = (PROT_READ | PROT_WRITE
| ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
-#if COLORING_INCREMENT != 0
- /* Add one more page for stack coloring. Don't do it for stacks
- with 16 times pagesize or larger. This might just cause
- unnecessary misalignment. */
- if (size <= 16 * pagesize_m1)
- size += pagesize_m1 + 1;
-#endif
-
/* Adjust the stack size for alignment. */
size &= ~__static_tls_align_m1;
assert (size != 0);
/* Make sure the size of the stack is enough for the guard and
eventually the thread descriptor. */
guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
+ if (guardsize < attr->guardsize || size + guardsize < guardsize)
+ /* Arithmetic overflow. */
+ return EINVAL;
+ size += guardsize;
if (__builtin_expect (size < ((guardsize + __static_tls_size
+ MINIMAL_REST_STACK + pagesize_m1)
& ~pagesize_m1),
size += pagesize_m1 + 1;
#endif
- mem = mmap (NULL, size, prot,
- MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+ /* If a guard page is required, avoid committing memory by first
+ allocate with PROT_NONE and then reserve with required permission
+ excluding the guard page. */
+ mem = __mmap (NULL, size, (guardsize == 0) ? prot : PROT_NONE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
if (__glibc_unlikely (mem == MAP_FAILED))
return errno;
So we can never get a null pointer back from mmap. */
assert (mem != NULL);
-#if COLORING_INCREMENT != 0
- /* Atomically increment NCREATED. */
- unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
-
- /* We chose the offset for coloring by incrementing it for
- every new thread by a fixed amount. The offset used
- module the page size. Even if coloring would be better
- relative to higher alignment values it makes no sense to
- do it since the mmap() interface does not allow us to
- specify any alignment for the returned memory block. */
- size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
-
- /* Make sure the coloring offsets does not disturb the alignment
- of the TCB and static TLS block. */
- if (__glibc_unlikely ((coloring & __static_tls_align_m1) != 0))
- coloring = (((coloring + __static_tls_align_m1)
- & ~(__static_tls_align_m1))
- & ~pagesize_m1);
-#else
- /* Unless specified we do not make any adjustments. */
-# define coloring 0
-#endif
-
/* Place the thread descriptor at the end of the stack. */
#if TLS_TCB_AT_TP
- pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
+ pd = (struct pthread *) ((char *) mem + size) - 1;
#elif TLS_DTV_AT_TP
- pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
+ pd = (struct pthread *) ((((uintptr_t) mem + size
- __static_tls_size)
& ~__static_tls_align_m1)
- TLS_PRE_TCB_SIZE);
#endif
+ /* Now mprotect the required region excluding the guard area. */
+ if (__glibc_likely (guardsize > 0))
+ {
+ char *guard = guard_position (mem, size, guardsize, pd,
+ pagesize_m1);
+ if (setup_stack_prot (mem, size, guard, guardsize, prot) != 0)
+ {
+ __munmap (mem, size);
+ return errno;
+ }
+ }
+
/* Remember the stack-related values. */
pd->stackblock = mem;
pd->stackblock_size = size;
+ /* Update guardsize for newly allocated guardsize to avoid
+ an mprotect in guard resize below. */
+ pd->guardsize = guardsize;
/* We allocated the first block thread-specific data array.
This address will not change for the lifetime of this
__pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
#endif
-#ifndef __ASSUME_PRIVATE_FUTEX
- /* The thread must know when private futexes are supported. */
- pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
- header.private_futex);
-#endif
-
#ifdef NEED_DL_SYSINFO
SETUP_THREAD_SYSINFO (pd);
#endif
assert (errno == ENOMEM);
/* Free the stack memory we just allocated. */
- (void) munmap (mem, size);
+ (void) __munmap (mem, size);
return errno;
}
if (err != 0)
{
/* Free the stack memory we just allocated. */
- (void) munmap (mem, size);
+ (void) __munmap (mem, size);
return err;
}
/* Create or resize the guard area if necessary. */
if (__glibc_unlikely (guardsize > pd->guardsize))
{
-#ifdef NEED_SEPARATE_REGISTER_STACK
- char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
-#elif _STACK_GROWS_DOWN
- char *guard = mem;
-#elif _STACK_GROWS_UP
- char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
-#endif
- if (mprotect (guard, guardsize, PROT_NONE) != 0)
+ char *guard = guard_position (mem, size, guardsize, pd,
+ pagesize_m1);
+ if (__mprotect (guard, guardsize, PROT_NONE) != 0)
{
mprot_error:
lll_lock (stack_cache_lock, LLL_PRIVATE);
of memory caused problems we better do not use it
anymore. Uh, and we ignore possible errors. There
is nothing we could do. */
- (void) munmap (mem, size);
+ (void) __munmap (mem, size);
return errno;
}
char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
if (oldguard < guard
- && mprotect (oldguard, guard - oldguard, prot) != 0)
+ && __mprotect (oldguard, guard - oldguard, prot) != 0)
goto mprot_error;
- if (mprotect (guard + guardsize,
+ if (__mprotect (guard + guardsize,
oldguard + pd->guardsize - guard - guardsize,
prot) != 0)
goto mprot_error;
#elif _STACK_GROWS_DOWN
- if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
+ if (__mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
prot) != 0)
goto mprot_error;
#elif _STACK_GROWS_UP
- if (mprotect ((char *) pd - pd->guardsize,
- pd->guardsize - guardsize, prot) != 0)
+ char *new_guard = (char *)(((uintptr_t) pd - guardsize)
+ & ~pagesize_m1);
+ char *old_guard = (char *)(((uintptr_t) pd - pd->guardsize)
+ & ~pagesize_m1);
+ /* The guard size difference might be > 0, but once rounded
+ to the nearest page the size difference might be zero. */
+ if (new_guard > old_guard
+ && __mprotect (old_guard, new_guard - old_guard, prot) != 0)
goto mprot_error;
#endif
- offsetof (pthread_mutex_t,
__data.__list.__next));
pd->robust_head.list_op_pending = NULL;
-#ifdef __PTHREAD_MUTEX_HAVE_PREV
+#if __PTHREAD_MUTEX_HAVE_PREV
pd->robust_prev = &pd->robust_head;
#endif
pd->robust_head.list = &pd->robust_head;
void
-internal_function
__deallocate_stack (struct pthread *pd)
{
lll_lock (stack_cache_lock, LLL_PRIVATE);
int
-internal_function
__make_stacks_executable (void **stack_endp)
{
/* First the main thread's stack. */
#ifdef SIGSETXID
static void
-internal_function
setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
{
int ch;
static void
-internal_function
setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t)
{
int ch;
static int
-internal_function
setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
{
if ((t->cancelhandling & SETXID_BITMASK) == 0)
if (olderror == error)
break;
if (olderror != -1)
- /* Mismatch between current and previous results. */
- abort ();
+ {
+ /* Mismatch between current and previous results. Save the
+ error value to memory so that is not clobbered by the
+ abort function and preserved in coredumps. */
+ volatile int xid_err __attribute__((unused)) = error;
+ abort ();
+ }
}
while (atomic_compare_and_exchange_bool_acq (&cmdp->error, error, -1));
}
# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
# endif
- /* Fill in the DTV slot so that a later LD/GD access will find it. */
- dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
- dtv[map->l_tls_modid].pointer.to_free = NULL;
- dtv[map->l_tls_modid].pointer.val = dest;
-
/* Initialize the memory. */
memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
'\0', map->l_tls_blocksize - map->l_tls_initimage_size);
projects / thirdparty / glibc.git / blobdiff