+2015-07-10 Torvald Riegel <triegel@redhat.com>
+
+ * sysdeps/nptl/futex-internal.h: New file.
+ * sysdeps/nacl/futex-internal.h: New file.
+ * sysdeps/unix/sysv/linux/futex-internal.h: New file.
+ * nptl/allocatestack.c (setxid_mark_thread): Use futex wrappers with
+ error checking.
+ (setxid_unmark_thread): Likewise.
+ (__nptl_setxid): Likewise.
+ (__wait_lookup_done): Likewise.
+ * nptl/cancellation.c (__pthread_disable_asynccancel): Likewise.
+ * nptl/nptl-init.c (sighandler_setxid): Likewise.
+ * nptl/pthread_create.c (START_THREAD_DEFN): Likewise.
+ * nptl/pthread_once.c (clear_once_control): Likewise.
+ (__pthread_once_slow): Likewise.
+ * nptl/unregister-atfork.c (__unregister_atfork): Likewise.
+ * sysdeps/nacl/exit-thread.h (__exit_thread): Likewise.
+ * sysdeps/nptl/aio_misc.h (AIO_MISC_NOTIFY, AIO_MISC_WAIT): Likewise.
+ * sysdeps/nptl/fork.c (__libc_fork): Likewise.
+ * sysdeps/nptl/gai_misc.h (GAI_MISC_NOTIFY, GAI_MISC_WAIT): Likewise.
+ * nptl/pthread_rwlock_rdlock.c (__pthread_rwlock_rdlock_slow):
+ Likewise.
+ (__pthread_rwlock_rdlock): Likewise.
+ * nptl/pthread_rwlock_timedrdlock.c (pthread_rwlock_timedrdlock):
+ Likewise.
+ * nptl/pthread_rwlock_timedwrlock.c (pthread_rwlock_timedwrlock):
+ Likewise.
+ * nptl/pthread_rwlock_tryrdlock.c (__pthread_rwlock_tryrdlock):
+ Likewise.
+ * nptl/pthread_rwlock_unlock.c (__pthread_rwlock_unlock): Likewise.
+ * nptl/pthread_rwlock_wrlock.c (__pthread_rwlock_wrlock_slow:
+ Likewise.
+ * nptl/pthread_rwlock_init (__pthread_rwlock_init): Remove
+ __ASSUME_PRIVATE_FUTEX check.
+ * nptl/pthread_rwlockattr_setpshared (pthread_rwlockattr_setpshared):
+ Check that shared futexes are supported.
+ * nptl/pthread_barrier_wait.c (pthread_barrier_wait): Use futex
+ wrappers with error checking.
+ * nptl/pthread_barrier_init.c (pthread_barrier_init): Add comments,
+ remove attribute sanity check and __ASSUME_PRIVATE_FUTEX check.
+ * nptl/pthread_barrierattr_setpshared.c
+ (pthread_barrierattr_setpshared): Check that shared futexes are
+ supported.
+ * nptl/pthread_condattr_setpshared.c (pthread_condattr_setpshared):
+ Likewise.
+ * nptl/pthread_mutexattr_setpshared.c (pthread_mutexattr_setpshared):
+ Likewise.
+ * nptl/sem_init.c (futex_private_if_supported): Remove.
+ (__new_sem_init): Adapt and check that shared futexes are supported.
+ * nptl/sem_open.c (sem_open): Likewise.
+ * nptl/sem_post.c (futex_wake): Remove.
+ * nptl/sem_waitcommon.c (futex_abstimed_wait, futex_wake): Remove.
+ (do_futex_wait): Use futex wrappers with error checking.
+ * nptl/sem_wait.c: Include lowlevellock.h.
+ * sysdeps/sparc/nptl/pthread_barrier_init.c (__pthread_barrier_init):
+ Use futex_supports_pshared.
+ * sysdeps/sparc/nptl/pthread_barrier_wait.c (pthread_barrier_wait):
+ Use futex wrappers with error checking.
+ * sysdeps/sparc/sparc32/pthread_barrier_wait.c (pthread_barrier_wait):
+ Likewise.
+ * sysdeps/sparc/sparc32/sem_init.c (futex_private_if_supported): Remove.
+ * sysdeps/sparc/sparc32/sem_post.c (futex_wake): Likewise.
+ * sysdeps/sparc/sparc32/sem_open.c (sem_open): Use FUTEX_SHARED.
+ * sysdeps/sparc/sparc32/sem_waitcommon.c (futex_abstimed_wait): Remove.
+ (futex_wake): Likewise.
+ (sem_assume_only_signals_cause_futex_EINTR): Likewise.
+ (do_futex_wait): Use futex wrappers with error checking.
+ (__new_sem_wait_slow): Update EINTR handling.
+ * sysdeps/sparc/sparc32/sem_wait.c: Include lowlevellock.h.
+
2015-07-09 Martin Sebor <msebor@redhat.com>
[BZ #18435]
#include <tls.h>
#include <list.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <kernel-features.h>
#include <stack-aliasing.h>
if (t->setxid_futex == -1
&& ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1))
do
- lll_futex_wait (&t->setxid_futex, -2, LLL_PRIVATE);
+ futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE);
while (t->setxid_futex == -2);
/* Don't let the thread exit before the setxid handler runs. */
if ((ch & SETXID_BITMASK) == 0)
{
t->setxid_futex = 1;
- lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE);
+ futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
}
return;
}
/* Release the futex just in case. */
t->setxid_futex = 1;
- lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE);
+ futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
}
int cur = cmdp->cntr;
while (cur != 0)
{
- lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
+ futex_wait_simple ((unsigned int *) &cmdp->cntr, cur,
+ FUTEX_PRIVATE);
cur = cmdp->cntr;
}
}
continue;
do
- lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
+ futex_wait_simple ((unsigned int *) gscope_flagp,
+ THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
continue;
do
- lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
+ futex_wait_simple ((unsigned int *) gscope_flagp,
+ THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
#include <setjmp.h>
#include <stdlib.h>
#include "pthreadP.h"
+#include <futex-internal.h>
/* The next two functions are similar to pthread_setcanceltype() but
while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK))
== CANCELING_BITMASK, 0))
{
- lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE);
+ futex_wait_simple ((unsigned int *) &self->cancelhandling, newval,
+ FUTEX_PRIVATE);
newval = THREAD_GETMEM (self, cancelhandling);
}
}
#include <shlib-compat.h>
#include <smp.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <kernel-features.h>
#include <libc-internal.h>
#include <pthread-pids.h>
/* And release the futex. */
self->setxid_futex = 1;
- lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE);
+ futex_wake (&self->setxid_futex, 1, FUTEX_PRIVATE);
if (atomic_decrement_val (&__xidcmd->cntr) == 0)
- lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE);
+ futex_wake ((unsigned int *) &__xidcmd->cntr, 1, FUTEX_PRIVATE);
}
#endif
{
struct pthread_barrier *ibarrier;
+ /* XXX EINVAL is not specified by POSIX as a possible error code. */
if (__glibc_unlikely (count == 0))
return EINVAL;
? iattr = (struct pthread_barrierattr *) attr
: &default_barrierattr);
- if (iattr->pshared != PTHREAD_PROCESS_PRIVATE
- && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0))
- /* Invalid attribute. */
- return EINVAL;
-
ibarrier = (struct pthread_barrier *) barrier;
/* Initialize the individual fields. */
ibarrier->init_count = count;
ibarrier->curr_event = 0;
-#ifdef __ASSUME_PRIVATE_FUTEX
+ /* XXX Don't use FUTEX_SHARED or FUTEX_PRIVATE as long as there are still
+ assembly implementations that expect the value determined below. */
ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE
? 0 : FUTEX_PRIVATE_FLAG);
-#else
- ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE
- ? 0 : THREAD_GETMEM (THREAD_SELF,
- header.private_futex));
-#endif
return 0;
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthreadP.h>
{
struct pthread_barrier *ibarrier = (struct pthread_barrier *) barrier;
int result = 0;
+ int lll_private = ibarrier->private ^ FUTEX_PRIVATE_FLAG;
+ int futex_private = (lll_private == LLL_PRIVATE
+ ? FUTEX_PRIVATE : FUTEX_SHARED);
/* Make sure we are alone. */
- lll_lock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+ lll_lock (ibarrier->lock, lll_private);
/* One more arrival. */
--ibarrier->left;
++ibarrier->curr_event;
/* Wake up everybody. */
- lll_futex_wake (&ibarrier->curr_event, INT_MAX,
- ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+ futex_wake (&ibarrier->curr_event, INT_MAX, futex_private);
/* This is the thread which finished the serialization. */
result = PTHREAD_BARRIER_SERIAL_THREAD;
unsigned int event = ibarrier->curr_event;
/* Before suspending, make the barrier available to others. */
- lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+ lll_unlock (ibarrier->lock, lll_private);
/* Wait for the event counter of the barrier to change. */
do
- lll_futex_wait (&ibarrier->curr_event, event,
- ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+ futex_wait_simple (&ibarrier->curr_event, event, futex_private);
while (event == ibarrier->curr_event);
}
/* If this was the last woken thread, unlock. */
if (atomic_increment_val (&ibarrier->left) == init_count)
/* We are done. */
- lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+ lll_unlock (ibarrier->lock, lll_private);
return result;
}
#include <errno.h>
#include "pthreadP.h"
+#include <futex-internal.h>
int
{
struct pthread_barrierattr *iattr;
- if (pshared != PTHREAD_PROCESS_PRIVATE
- && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
- return EINVAL;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ return err;
iattr = (struct pthread_barrierattr *) attr;
#include <errno.h>
#include <pthreadP.h>
+#include <futex-internal.h>
int
pthread_condattr_setpshared (attr, pshared)
pthread_condattr_t *attr;
int pshared;
{
- if (pshared != PTHREAD_PROCESS_PRIVATE
- && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
- return EINVAL;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ return err;
int *valuep = &((struct pthread_condattr *) attr)->value;
#include <kernel-features.h>
#include <exit-thread.h>
#include <default-sched.h>
+#include <futex-internal.h>
#include <shlib-compat.h>
/* Allow setxid from now onwards. */
if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2))
- lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
+ futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE);
#ifdef __NR_set_robust_list
# ifndef __ASSUME_SET_ROBUST_LIST
this->__list.__next = NULL;
atomic_or (&this->__lock, FUTEX_OWNER_DIED);
- lll_futex_wake (&this->__lock, 1, /* XYZ */ LLL_SHARED);
+ futex_wake ((unsigned int *) &this->__lock, 1,
+ /* XYZ */ FUTEX_SHARED);
}
while (robust != (void *) &pd->robust_head);
}
/* Some other thread might call any of the setXid functions and expect
us to reply. In this case wait until we did that. */
do
- lll_futex_wait (&pd->setxid_futex, 0, LLL_PRIVATE);
+ /* XXX This differs from the typical futex_wait_simple pattern in that
+ the futex_wait condition (setxid_futex) is different from the
+ condition used in the surrounding loop (cancelhandling). We need
+ to check and document why this is correct. */
+ futex_wait_simple (&pd->setxid_futex, 0, FUTEX_PRIVATE);
while (pd->cancelhandling & SETXID_BITMASK);
/* Reset the value so that the stack can be reused. */
stillborn thread. */
if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0)
== -2))
- lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
+ futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE);
/* Free the resources. */
__deallocate_stack (pd);
#include <errno.h>
#include <pthreadP.h>
+#include <futex-internal.h>
int
{
struct pthread_mutexattr *iattr;
- if (pshared != PTHREAD_PROCESS_PRIVATE
- && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
- return EINVAL;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ return err;
iattr = (struct pthread_mutexattr *) attr;
<http://www.gnu.org/licenses/>. */
#include "pthreadP.h"
-#include <lowlevellock.h>
+#include <futex-internal.h>
#include <atomic.h>
get interrupted (see __pthread_once), so all we need to relay to other
threads is the state being reset again. */
atomic_store_relaxed (once_control, 0);
- lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
+ futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
}
is set and __PTHREAD_ONCE_DONE is not. */
if (val == newval)
{
- /* Same generation, some other thread was faster. Wait. */
- lll_futex_wait (once_control, newval, LLL_PRIVATE);
+ /* Same generation, some other thread was faster. Wait and
+ retry. */
+ futex_wait_simple ((unsigned int *) once_control,
+ (unsigned int) newval, FUTEX_PRIVATE);
continue;
}
}
atomic_store_release (once_control, __PTHREAD_ONCE_DONE);
/* Wake up all other threads. */
- lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
+ futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
break;
}
If the pshared value is in locking functions XORed with avail
we get the expected result. */
-#ifdef __ASSUME_PRIVATE_FUTEX
rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
? 0 : FUTEX_PRIVATE_FLAG);
-#else
- rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
- ? 0
- : THREAD_GETMEM (THREAD_SELF,
- header.private_futex));
-#endif
return 0;
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
{
int result = 0;
bool wake = false;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
/* Lock is taken in caller. */
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- /* Wait for the writer to finish. */
- lll_futex_wait (&rwlock->__data.__readers_wakeup, waitval,
- rwlock->__data.__shared);
+ /* Wait for the writer to finish. We do not check the return value
+ because we decide how to continue based on the state of the rwlock. */
+ futex_wait_simple (&rwlock->__data.__readers_wakeup, waitval,
+ futex_shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
if (wake)
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
{
int result = 0;
bool wake = false;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
LIBC_PROBE (rdlock_entry, 1, rwlock);
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
if (wake)
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthread.h>
#include <pthreadP.h>
#include <sys/time.h>
-#include <kernel-features.h>
#include <stdbool.h>
{
int result = 0;
bool wake = false;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
/* Make sure we are alone. */
lll_lock(rwlock->__data.__lock, rwlock->__data.__shared);
break;
}
- /* Work around the fact that the kernel rejects negative timeout values
- despite them being valid. */
- if (__glibc_unlikely (abstime->tv_sec < 0))
- {
- result = ETIMEDOUT;
- break;
- }
-
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
- || !defined lll_futex_timed_wait_bitset)
- /* Get the current time. So far we support only one clock. */
- struct timeval tv;
- (void) __gettimeofday (&tv, NULL);
-
- /* Convert the absolute timeout value to a relative timeout. */
- struct timespec rt;
- rt.tv_sec = abstime->tv_sec - tv.tv_sec;
- rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
- if (rt.tv_nsec < 0)
- {
- rt.tv_nsec += 1000000000;
- --rt.tv_sec;
- }
- /* Did we already time out? */
- if (rt.tv_sec < 0)
- {
- /* Yep, return with an appropriate error. */
- result = ETIMEDOUT;
- break;
- }
-#endif
-
/* Remember that we are a reader. */
if (++rwlock->__data.__nr_readers_queued == 0)
{
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- /* Wait for the writer to finish. */
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
- || !defined lll_futex_timed_wait_bitset)
- err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup,
- waitval, &rt, rwlock->__data.__shared);
-#else
- err = lll_futex_timed_wait_bitset (&rwlock->__data.__readers_wakeup,
- waitval, abstime,
- FUTEX_CLOCK_REALTIME,
- rwlock->__data.__shared);
-#endif
+ /* Wait for the writer to finish. We handle ETIMEDOUT below; on other
+ return values, we decide how to continue based on the state of the
+ rwlock. */
+ err = futex_abstimed_wait (&rwlock->__data.__readers_wakeup, waitval,
+ abstime, futex_shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
--rwlock->__data.__nr_readers_queued;
/* Did the futex call time out? */
- if (err == -ETIMEDOUT)
+ if (err == ETIMEDOUT)
{
/* Yep, report it. */
result = ETIMEDOUT;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
if (wake)
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthread.h>
#include <pthreadP.h>
#include <sys/time.h>
-#include <kernel-features.h>
#include <stdbool.h>
{
int result = 0;
bool wake_readers = false;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
/* Make sure we are alone. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
break;
}
- /* Work around the fact that the kernel rejects negative timeout values
- despite them being valid. */
- if (__glibc_unlikely (abstime->tv_sec < 0))
- {
- result = ETIMEDOUT;
- break;
- }
-
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
- || !defined lll_futex_timed_wait_bitset)
- /* Get the current time. So far we support only one clock. */
- struct timeval tv;
- (void) __gettimeofday (&tv, NULL);
-
- /* Convert the absolute timeout value to a relative timeout. */
- struct timespec rt;
- rt.tv_sec = abstime->tv_sec - tv.tv_sec;
- rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
- if (rt.tv_nsec < 0)
- {
- rt.tv_nsec += 1000000000;
- --rt.tv_sec;
- }
- /* Did we already time out? */
- if (rt.tv_sec < 0)
- {
- result = ETIMEDOUT;
- break;
- }
-#endif
-
/* Remember that we are a writer. */
if (++rwlock->__data.__nr_writers_queued == 0)
{
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- /* Wait for the writer or reader(s) to finish. */
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
- || !defined lll_futex_timed_wait_bitset)
- err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup,
- waitval, &rt, rwlock->__data.__shared);
-#else
- err = lll_futex_timed_wait_bitset (&rwlock->__data.__writer_wakeup,
- waitval, abstime,
- FUTEX_CLOCK_REALTIME,
- rwlock->__data.__shared);
-#endif
+ /* Wait for the writer or reader(s) to finish. We handle ETIMEDOUT
+ below; on other return values, we decide how to continue based on
+ the state of the rwlock. */
+ err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval,
+ abstime, futex_shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
--rwlock->__data.__nr_writers_queued;
/* Did the futex call time out? */
- if (err == -ETIMEDOUT)
+ if (err == ETIMEDOUT)
{
result = ETIMEDOUT;
/* If we prefer writers, it can have happened that readers blocked
/* Might be required after timeouts. */
if (wake_readers)
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
#include <errno.h>
#include "pthreadP.h"
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <elide.h>
#include <stdbool.h>
{
int result = EBUSY;
bool wake = false;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
rwlock->__data.__lock == 0
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
if (wake)
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
int
__pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
+
LIBC_PROBE (rwlock_unlock, 1, rwlock);
if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
{
++rwlock->__data.__writer_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- lll_futex_wake (&rwlock->__data.__writer_wakeup, 1,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared);
return 0;
}
else if (rwlock->__data.__nr_readers_queued)
{
++rwlock->__data.__readers_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
- rwlock->__data.__shared);
+ futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
+ futex_shared);
return 0;
}
}
#include <errno.h>
#include <sysdep.h>
#include <lowlevellock.h>
+#include <futex-internal.h>
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
__pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock)
{
int result = 0;
+ int futex_shared =
+ rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
/* Caller has taken the lock. */
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
- /* Wait for the writer or reader(s) to finish. */
- lll_futex_wait (&rwlock->__data.__writer_wakeup, waitval,
- rwlock->__data.__shared);
+ /* Wait for the writer or reader(s) to finish. We do not check the
+ return value because we decide how to continue based on the state of
+ the rwlock. */
+ futex_wait_simple (&rwlock->__data.__writer_wakeup, waitval,
+ futex_shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
#include <errno.h>
#include "pthreadP.h"
+#include <futex-internal.h>
int
{
struct pthread_rwlockattr *iattr;
- if (pshared != PTHREAD_PROCESS_SHARED
- && __builtin_expect (pshared != PTHREAD_PROCESS_PRIVATE, 0))
- return EINVAL;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ return err;
iattr = (struct pthread_rwlockattr *) attr;
#include <shlib-compat.h>
#include "semaphoreP.h"
#include <kernel-features.h>
-
-/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported;
- returns FUTEX_SHARED otherwise.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline int
-futex_private_if_supported (int pshared)
-{
- if (pshared != 0)
- return LLL_SHARED;
-#ifdef __ASSUME_PRIVATE_FUTEX
- return LLL_PRIVATE;
-#else
- return THREAD_GETMEM (THREAD_SELF, header.private_futex)
- ^ FUTEX_PRIVATE_FLAG;
-#endif
-}
+#include <futex-internal.h>
int
__set_errno (EINVAL);
return -1;
}
+ pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ {
+ __set_errno (err);
+ return -1;
+ }
/* Map to the internal type. */
struct new_sem *isem = (struct new_sem *) sem;
isem->nwaiters = 0;
#endif
- isem->private = futex_private_if_supported (pshared);
+ isem->private = (pshared == PTHREAD_PROCESS_PRIVATE
+ ? FUTEX_PRIVATE : FUTEX_SHARED);
return 0;
}
#include <sys/stat.h>
#include "semaphoreP.h"
#include <shm-directory.h>
+#include <futex-internal.h>
/* Comparison function for search of existing mapping. */
int fd;
sem_t *result;
+ /* Check that shared futexes are supported. */
+ int err = futex_supports_pshared (PTHREAD_PROCESS_SHARED);
+ if (err != 0)
+ {
+ __set_errno (err);
+ return SEM_FAILED;
+ }
+
/* Create the name of the final file in local variable SHM_NAME. */
SHM_GET_NAME (EINVAL, SEM_FAILED, SEM_SHM_PREFIX);
sem.newsem.nwaiters = 0;
#endif
/* This always is a shared semaphore. */
- sem.newsem.private = LLL_SHARED;
+ sem.newsem.private = FUTEX_SHARED;
/* Initialize the remaining bytes as well. */
memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0',
#include <atomic.h>
#include <errno.h>
#include <sysdep.h>
-#include <lowlevellock.h>
+#include <lowlevellock.h> /* lll_futex* used by the old code. */
+#include <futex-internal.h>
#include <internaltypes.h>
#include <semaphore.h>
#include <shlib-compat.h>
-/* Wrapper for lll_futex_wake, with error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
- int res = lll_futex_wake (futex, processes_to_wake, private);
- /* No error. Ignore the number of woken processes. */
- if (res >= 0)
- return;
- switch (res)
- {
- case -EFAULT: /* Could have happened due to memory reuse. */
- case -EINVAL: /* Could be either due to incorrect alignment (a bug in
- glibc or in the application) or due to memory being
- reused for a PI futex. We cannot distinguish between the
- two causes, and one of them is correct use, so we do not
- act in this case. */
- return;
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
/* See sem_wait for an explanation of the algorithm. */
int
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#include <lowlevellock.h> /* lll_futex* used by the old code. */
#include "sem_waitcommon.c"
int
#include <kernel-features.h>
#include <errno.h>
#include <sysdep.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
#include <internaltypes.h>
#include <semaphore.h>
#include <sys/time.h>
#include <shlib-compat.h>
#include <atomic.h>
-/* Wrapper for lll_futex_wait with absolute timeout and error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline int
-futex_abstimed_wait (unsigned int* futex, unsigned int expected,
- const struct timespec* abstime, int private, bool cancel)
-{
- int err, oldtype;
- if (abstime == NULL)
- {
- if (cancel)
- oldtype = __pthread_enable_asynccancel ();
- err = lll_futex_wait (futex, expected, private);
- if (cancel)
- __pthread_disable_asynccancel (oldtype);
- }
- else
- {
-#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \
- && defined lll_futex_timed_wait_bitset)
- /* The Linux kernel returns EINVAL for this, but in userspace
- such a value is valid. */
- if (abstime->tv_sec < 0)
- return ETIMEDOUT;
-#else
- struct timeval tv;
- struct timespec rt;
- int sec, nsec;
-
- /* Get the current time. */
- __gettimeofday (&tv, NULL);
-
- /* Compute relative timeout. */
- sec = abstime->tv_sec - tv.tv_sec;
- nsec = abstime->tv_nsec - tv.tv_usec * 1000;
- if (nsec < 0)
- {
- nsec += 1000000000;
- --sec;
- }
-
- /* Already timed out? */
- if (sec < 0)
- return ETIMEDOUT;
-
- /* Do wait. */
- rt.tv_sec = sec;
- rt.tv_nsec = nsec;
-#endif
- if (cancel)
- oldtype = __pthread_enable_asynccancel ();
-#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \
- && defined lll_futex_timed_wait_bitset)
- err = lll_futex_timed_wait_bitset (futex, expected, abstime,
- FUTEX_CLOCK_REALTIME, private);
-#else
- err = lll_futex_timed_wait (futex, expected, &rt, private);
-#endif
- if (cancel)
- __pthread_disable_asynccancel (oldtype);
- }
- switch (err)
- {
- case 0:
- case -EAGAIN:
- case -EINTR:
- case -ETIMEDOUT:
- return -err;
-
- case -EFAULT: /* Must have been caused by a glibc or application bug. */
- case -EINVAL: /* Either due to wrong alignment or due to the timeout not
- being normalized. Must have been caused by a glibc or
- application bug. */
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
-/* Wrapper for lll_futex_wake, with error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
- int res = lll_futex_wake (futex, processes_to_wake, private);
- /* No error. Ignore the number of woken processes. */
- if (res >= 0)
- return;
- switch (res)
- {
- case -EFAULT: /* Could have happened due to memory reuse. */
- case -EINVAL: /* Could be either due to incorrect alignment (a bug in
- glibc or in the application) or due to memory being
- reused for a PI futex. We cannot distinguish between the
- two causes, and one of them is correct use, so we do not
- act in this case. */
- return;
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
/* The semaphore provides two main operations: sem_post adds a token to the
semaphore; sem_wait grabs a token from the semaphore, potentially waiting
int err;
#if __HAVE_64B_ATOMICS
- err = futex_abstimed_wait ((unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0,
- abstime, sem->private, true);
+ err = futex_abstimed_wait_cancelable (
+ (unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, abstime,
+ sem->private);
#else
- err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime,
- sem->private, true);
+ err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK,
+ abstime, sem->private);
#endif
return err;
#include <stdlib.h>
#include <fork.h>
#include <atomic.h>
+#include <futex-internal.h>
void
atomic_decrement (&deleted->handler->refcntr);
unsigned int val;
while ((val = deleted->handler->refcntr) != 0)
- lll_futex_wait (&deleted->handler->refcntr, val, LLL_PRIVATE);
+ futex_wait_simple (&deleted->handler->refcntr, val, FUTEX_PRIVATE);
deleted = deleted->next;
}
#include <assert.h>
#include <atomic.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
#include <nacl-interfaces.h>
#include <nptl/pthreadP.h>
assert (NACL_EXITING_TID > 0);
atomic_store_relaxed (&pd->tid, NACL_EXITING_TID);
- lll_futex_wake (&pd->tid, 1, LLL_PRIVATE);
+ futex_wake ((unsigned int *) &pd->tid, 1, FUTEX_PRIVATE);
}
/* This clears PD->tid some time after the thread stack can never
--- /dev/null
+/* futex operations for glibc-internal use. NaCl version.
+ Copyright (C) 2014-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef FUTEX_INTERNAL_H
+#define FUTEX_INTERNAL_H
+
+#include <sysdeps/nptl/futex-internal.h>
+#include <errno.h>
+#include <lowlevellock-futex.h>
+#include <nacl-interfaces.h>
+#include <nptl/pthreadP.h>
+
+/* See sysdeps/nptl/futex-internal.h for documentation; this file only
+ contains NaCl-specific comments.
+
+ There is no support yet for shared futexes nor for exact relative
+ timeouts. */
+
+/* See sysdeps/nptl/futex-internal.h for constraints on the value of the
+ FUTEX_PRIVATE and FUTEX_SHARED constants.
+ Shared futexes are not yet supported, and we never allow clients to
+ actually request shared futexes. Therefore, we do not need a different
+ value. */
+#undef FUTEX_SHARED
+#define FUTEX_SHARED FUTEX_PRIVATE
+
+/* FUTEX_SHARED is not yet supported. */
+static __always_inline int
+futex_supports_pshared (int pshared)
+{
+ if (__glibc_likely (pshared == PTHREAD_PROCESS_PRIVATE))
+ return 0;
+ else if (pshared == PTHREAD_PROCESS_SHARED)
+ return ENOTSUP;
+ else
+ return EINVAL;
+}
+
+/* Relative timeouts are only emulated via absolute timeouts using the
+ system clock. */
+static __always_inline bool
+futex_supports_exact_relative_timeouts (void)
+{
+ return false;
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_wait (unsigned int *futex_word, unsigned int expected, int private)
+{
+ int err = lll_futex_timed_wait (futex_word, expected, NULL, private);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ return -err;
+
+ case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_wait_cancelable (unsigned int *futex_word, unsigned int expected,
+ int private)
+{
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = lll_futex_timed_wait (futex_word, expected, NULL, private);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ return -err;
+
+ case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_reltimed_wait (unsigned int *futex_word, unsigned int expected,
+ const struct timespec *reltime, int private)
+{
+ int err = lll_futex_timed_wait (futex_word, expected, reltime, private);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_reltimed_wait_cancelable (unsigned int *futex_word,
+ unsigned int expected,
+ const struct timespec *reltime, int private)
+{
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = lll_futex_timed_wait (futex_word, expected, reltime, private);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_abstimed_wait (unsigned int *futex_word, unsigned int expected,
+ const struct timespec *abstime, int private)
+{
+ int err = __nacl_irt_futex.futex_wait_abs ((volatile int *) futex_word,
+ expected, abstime);
+ switch (err)
+ {
+ case 0:
+ case EAGAIN:
+ case EINTR:
+ case ETIMEDOUT:
+ return err;
+
+ case EFAULT: /* Must have been caused by a glibc or application bug. */
+ case EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_abstimed_wait_cancelable (unsigned int *futex_word,
+ unsigned int expected,
+ const struct timespec *abstime, int private)
+{
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = __nacl_irt_futex.futex_wait_abs ((volatile int *) futex_word,
+ expected, abstime);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case EAGAIN:
+ case EINTR:
+ case ETIMEDOUT:
+ return err;
+
+ case EFAULT: /* Must have been caused by a glibc or application bug. */
+ case EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline void
+futex_wake (unsigned int *futex_word, int processes_to_wake, int private)
+{
+ int res = lll_futex_wake (futex_word, processes_to_wake, private);
+ /* No error. Ignore the number of woken processes. */
+ if (res >= 0)
+ return;
+ switch (res)
+ {
+ case -EFAULT: /* Could have happened due to memory reuse. */
+ case -EINVAL: /* Could be either due to incorrect alignment (a bug in
+ glibc or in the application) or due to memory being
+ reused for a PI futex. We cannot distinguish between the
+ two causes, and one of them is correct use, so we do not
+ act in this case. */
+ return;
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+#endif /* futex-internal.h */
#include <assert.h>
#include <nptl/pthreadP.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
#define DONT_NEED_AIO_MISC_COND 1
#define AIO_MISC_NOTIFY(waitlist) \
do { \
if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \
- lll_futex_wake (waitlist->counterp, 1, LLL_PRIVATE); \
+ futex_wake ((unsigned int *) waitlist->counterp, 1, FUTEX_PRIVATE); \
} while (0)
#define AIO_MISC_WAIT(result, futex, timeout, cancel) \
int status; \
do \
{ \
- status = lll_futex_timed_wait (futexaddr, oldval, timeout, \
- LLL_PRIVATE); \
- if (status != -EWOULDBLOCK) \
+ status = futex_reltimed_wait ((unsigned int *) futexaddr, oldval, \
+ timeout, FUTEX_PRIVATE); \
+ if (status != EAGAIN) \
break; \
\
oldval = *futexaddr; \
if (cancel) \
LIBC_CANCEL_RESET (oldtype); \
\
- if (status == -EINTR) \
+ if (status == EINTR) \
result = EINTR; \
- else if (status == -ETIMEDOUT) \
+ else if (status == ETIMEDOUT) \
result = EAGAIN; \
else \
- assert (status == 0 || status == -EWOULDBLOCK); \
+ assert (status == 0 || status == EAGAIN); \
\
pthread_mutex_lock (&__aio_requests_mutex); \
} \
#include <nptl/pthreadP.h>
#include <fork.h>
#include <arch-fork.h>
+#include <futex-internal.h>
static void
if (atomic_decrement_and_test (&allp->handler->refcntr)
&& allp->handler->need_signal)
- lll_futex_wake (&allp->handler->refcntr, 1, LLL_PRIVATE);
+ futex_wake (&allp->handler->refcntr, 1, FUTEX_PRIVATE);
allp = allp->next;
}
--- /dev/null
+/* futex operations for glibc-internal use. Stub version; do not include
+ this file directly.
+ Copyright (C) 2014-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef STUB_FUTEX_INTERNAL_H
+#define STUB_FUTEX_INTERNAL_H
+
+#include <sys/time.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <libc-internal.h>
+
+/* This file defines futex operations used internally in glibc. A futex
+ consists of the so-called futex word in userspace, which is of type
+ unsigned int and represents an application-specific condition, and kernel
+ state associated with this particular futex word (e.g., wait queues). The
+ futex operations we provide are wrappers for the futex syscalls and add
+ glibc-specific error checking of the syscall return value. We abort on
+ error codes that are caused by bugs in glibc or in the calling application,
+ or when an error code is not known. We return error codes that can arise
+ in correct executions to the caller. Each operation calls out exactly the
+ return values that callers need to handle.
+
+ The private flag must be either FUTEX_PRIVATE or FUTEX_SHARED.
+ FUTEX_PRIVATE is always supported, and the implementation can internally
+ use FUTEX_SHARED when FUTEX_PRIVATE is requested. FUTEX_SHARED is not
+ necessarily supported (use futex_supports_pshared to detect this).
+
+ We expect callers to only use these operations if futexes and the
+ specific futex operations being used are supported (e.g., FUTEX_SHARED).
+
+ Given that waking other threads waiting on a futex involves concurrent
+ accesses to the futex word, you must use atomic operations to access the
+ futex word.
+
+ Both absolute and relative timeouts can be used. An absolute timeout
+ expires when the given specific point in time on the CLOCK_REALTIME clock
+ passes, or when it already has passed. A relative timeout expires when
+ the given duration of time on the CLOCK_MONOTONIC clock passes. Relative
+ timeouts may be imprecise (see futex_supports_exact_relative_timeouts).
+
+ Due to POSIX requirements on when synchronization data structures such
+ as mutexes or semaphores can be destroyed and due to the futex design
+ having separate fast/slow paths for wake-ups, we need to consider that
+ futex_wake calls might effectively target a data structure that has been
+ destroyed and reused for another object, or unmapped; thus, some
+ errors or spurious wake-ups can happen in correct executions that would
+ not be possible in a program using just a single futex whose lifetime
+ does not end before the program terminates. For background, see:
+ https://sourceware.org/ml/libc-alpha/2014-04/msg00075.html
+ https://lkml.org/lkml/2014/11/27/472 */
+
+/* Defined this way for interoperability with lowlevellock.
+ FUTEX_PRIVATE must be zero because the initializers for pthread_mutex_t,
+ pthread_rwlock_t, and pthread_cond_t initialize the respective field of
+ those structures to zero, and we want FUTEX_PRIVATE to be the default. */
+#define FUTEX_PRIVATE LLL_PRIVATE
+#define FUTEX_SHARED LLL_SHARED
+#if FUTEX_PRIVATE != 0
+# error FUTEX_PRIVATE must be equal to 0
+#endif
+
+/* Returns EINVAL if PSHARED is neither PTHREAD_PROCESS_PRIVATE nor
+ PTHREAD_PROCESS_SHARED; otherwise, returns 0 if PSHARED is supported, and
+ ENOTSUP if not. */
+static __always_inline int
+futex_supports_pshared (int pshared);
+
+/* Returns true if relative timeouts are robust to concurrent changes to the
+ system clock. If this returns false, relative timeouts can still be used
+ but might be effectively longer or shorter than requested. */
+static __always_inline bool
+futex_supports_exact_relative_timeouts (void);
+
+/* Atomically wrt other futex operations on the same futex, this blocks iff
+ the value *FUTEX_WORD matches the expected value. This is
+ semantically equivalent to:
+ l = <get lock associated with futex> (FUTEX_WORD);
+ wait_flag = <get wait_flag associated with futex> (FUTEX_WORD);
+ lock (l);
+ val = atomic_load_relaxed (FUTEX_WORD);
+ if (val != expected) { unlock (l); return EAGAIN; }
+ atomic_store_relaxed (wait_flag, true);
+ unlock (l);
+ // Now block; can time out in futex_time_wait (see below)
+ while (atomic_load_relaxed(wait_flag) && !<spurious wake-up>);
+
+ Note that no guarantee of a happens-before relation between a woken
+ futex_wait and a futex_wake is documented; however, this does not matter
+ in practice because we have to consider spurious wake-ups (see below),
+ and thus would not be able to reliably reason about which futex_wake woke
+ us.
+
+ Returns 0 if woken by a futex operation or spuriously. (Note that due to
+ the POSIX requirements mentioned above, we need to conservatively assume
+ that unrelated futex_wake operations could wake this futex; it is easiest
+ to just be prepared for spurious wake-ups.)
+ Returns EAGAIN if the futex word did not match the expected value.
+ Returns EINTR if waiting was interrupted by a signal.
+
+ Note that some previous code in glibc assumed the underlying futex
+ operation (e.g., syscall) to start with or include the equivalent of a
+ seq_cst fence; this allows one to avoid an explicit seq_cst fence before
+ a futex_wait call when synchronizing similar to Dekker synchronization.
+ However, we make no such guarantee here. */
+static __always_inline int
+futex_wait (unsigned int *futex_word, unsigned int expected, int private);
+
+/* Like futex_wait but does not provide any indication why we stopped waiting.
+ Thus, when this function returns, you have to always check FUTEX_WORD to
+ determine whether you need to continue waiting, and you cannot detect
+ whether the waiting was interrupted by a signal. Example use:
+ while (atomic_load_relaxed (&futex_word) == 23)
+ futex_wait_simple (&futex_word, 23, FUTEX_PRIVATE);
+ This is common enough to make providing this wrapper worthwhile. */
+static __always_inline void
+futex_wait_simple (unsigned int *futex_word, unsigned int expected,
+ int private)
+{
+ ignore_value (futex_wait (futex_word, expected, private));
+}
+
+
+/* Like futex_wait but is a POSIX cancellation point. */
+static __always_inline int
+futex_wait_cancelable (unsigned int *futex_word, unsigned int expected,
+ int private);
+
+/* Like futex_wait, but will eventually time out (i.e., stop being
+ blocked) after the duration of time provided (i.e., RELTIME) has
+ passed. The caller must provide a normalized RELTIME. RELTIME can also
+ equal NULL, in which case this function behaves equivalent to futex_wait.
+
+ Returns the same values as futex_wait under those same conditions;
+ additionally, returns ETIMEDOUT if the timeout expired.
+ */
+static __always_inline int
+futex_reltimed_wait (unsigned int* futex_word, unsigned int expected,
+ const struct timespec* reltime, int private);
+
+/* Like futex_reltimed_wait but is a POSIX cancellation point. */
+static __always_inline int
+futex_reltimed_wait_cancelable (unsigned int* futex_word,
+ unsigned int expected,
+ const struct timespec* reltime, int private);
+
+/* Like futex_reltimed_wait, but the provided timeout (ABSTIME) is an
+ absolute point in time; a call will time out after this point in time. */
+static __always_inline int
+futex_abstimed_wait (unsigned int* futex_word, unsigned int expected,
+ const struct timespec* abstime, int private);
+
+/* Like futex_reltimed_wait but is a POSIX cancellation point. */
+static __always_inline int
+futex_abstimed_wait_cancelable (unsigned int* futex_word,
+ unsigned int expected,
+ const struct timespec* abstime, int private);
+
+/* Atomically wrt other futex operations on the same futex, this unblocks the
+ specified number of processes, or all processes blocked on this futex if
+ there are fewer than the specified number. Semantically, this is
+ equivalent to:
+ l = <get lock associated with futex> (FUTEX_WORD);
+ lock (l);
+ for (res = 0; PROCESSES_TO_WAKE > 0; PROCESSES_TO_WAKE--, res++) {
+ if (<no process blocked on futex>) break;
+ wf = <get wait_flag of a process blocked on futex> (FUTEX_WORD);
+ // No happens-before guarantee with woken futex_wait (see above)
+ atomic_store_relaxed (wf, 0);
+ }
+ return res;
+
+ Note that we need to support futex_wake calls to past futexes whose memory
+ has potentially been reused due to POSIX' requirements on synchronization
+ object destruction (see above); therefore, we must not report or abort
+ on most errors. */
+static __always_inline void
+futex_wake (unsigned int* futex_word, int processes_to_wake, int private);
+
+/* Calls __libc_fatal with an error message. Convenience function for
+ concrete implementations of the futex interface. */
+static __always_inline __attribute__ ((__noreturn__)) void
+futex_fatal_error (void)
+{
+ __libc_fatal ("The futex facility returned an unexpected error code.");
+}
+
+#endif /* futex-internal.h */
#include <assert.h>
#include <signal.h>
#include <nptl/pthreadP.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
#define DONT_NEED_GAI_MISC_COND 1
#define GAI_MISC_NOTIFY(waitlist) \
do { \
if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \
- lll_futex_wake (waitlist->counterp, 1, LLL_PRIVATE); \
+ futex_wake ((unsigned int *) waitlist->counterp, 1, FUTEX_PRIVATE); \
} while (0)
#define GAI_MISC_WAIT(result, futex, timeout, cancel) \
int status; \
do \
{ \
- status = lll_futex_timed_wait (futexaddr, oldval, timeout, \
- LLL_PRIVATE); \
- if (status != -EWOULDBLOCK) \
+ status = futex_reltimed_wait ((unsigned int *) futexaddr, oldval, \
+ timeout, FUTEX_PRIVATE); \
+ if (status != EAGAIN) \
break; \
\
oldval = *futexaddr; \
if (cancel) \
LIBC_CANCEL_RESET (oldtype); \
\
- if (status == -EINTR) \
+ if (status == EINTR) \
result = EINTR; \
- else if (status == -ETIMEDOUT) \
+ else if (status == ETIMEDOUT) \
result = EAGAIN; \
else \
- assert (status == 0 || status == -EWOULDBLOCK); \
+ assert (status == 0 || status == EAGAIN); \
\
pthread_mutex_lock (&__gai_requests_mutex); \
} \
struct pthread_barrierattr *iattr = (struct pthread_barrierattr *) attr;
if (iattr != NULL)
{
- if (iattr->pshared != PTHREAD_PROCESS_PRIVATE
- && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0))
- /* Invalid attribute. */
- return EINVAL;
+ int err = futex_supports_pshared (iattr->pshared);
+ if (err != 0)
+ return err;
}
ibarrier = (union sparc_pthread_barrier *) barrier;
#include <lowlevellock.h>
#include <pthreadP.h>
#include <sparc-nptl.h>
+#include <futex-internal.h>
/* Wait on barrier. */
int
= (union sparc_pthread_barrier *) barrier;
int result = 0;
int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE;
+ int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE;
/* Make sure we are alone. */
lll_lock (ibarrier->b.lock, private);
++ibarrier->b.curr_event;
/* Wake up everybody. */
- lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, private);
+ futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private);
/* This is the thread which finished the serialization. */
result = PTHREAD_BARRIER_SERIAL_THREAD;
/* Wait for the event counter of the barrier to change. */
do
- lll_futex_wait (&ibarrier->b.curr_event, event, private);
+ futex_wait_simple (&ibarrier->b.curr_event, event, futex_private);
while (event == ibarrier->b.curr_event);
}
#include <lowlevellock.h>
#include <pthreadP.h>
#include <sparc-nptl.h>
+#include <futex-internal.h>
/* Wait on barrier. */
int
= (union sparc_pthread_barrier *) barrier;
int result = 0;
int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE;
+ int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE;
/* Make sure we are alone. */
lll_lock (ibarrier->b.lock, private);
++ibarrier->b.curr_event;
/* Wake up everybody. */
- lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, private);
+ futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private);
/* This is the thread which finished the serialization. */
result = PTHREAD_BARRIER_SERIAL_THREAD;
/* Wait for the event counter of the barrier to change. */
do
- lll_futex_wait (&ibarrier->b.curr_event, event, private);
+ futex_wait_simple (&ibarrier->b.curr_event, event, futex_private);
while (event == ibarrier->b.curr_event);
}
#include <semaphore.h>
#include <shlib-compat.h>
#include "semaphoreP.h"
-#include <kernel-features.h>
-
-/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported;
- returns FUTEX_SHARED otherwise.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline int
-futex_private_if_supported (int pshared)
-{
- if (pshared != 0)
- return LLL_SHARED;
-#ifdef __ASSUME_PRIVATE_FUTEX
- return LLL_PRIVATE;
-#else
- return THREAD_GETMEM (THREAD_SELF, header.private_futex)
- ^ FUTEX_PRIVATE_FLAG;
-#endif
-}
+#include <futex-internal.h>
int
return -1;
}
+ pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE;
+ int err = futex_supports_pshared (pshared);
+ if (err != 0)
+ {
+ __set_errno (err);
+ return -1;
+ }
+
/* Map to the internal type. */
struct new_sem *isem = (struct new_sem *) sem;
isem->pad = 0;
isem->nwaiters = 0;
- isem->private = futex_private_if_supported (pshared);
+ isem->private = (pshared == PTHREAD_PROCESS_PRIVATE
+ ? FUTEX_PRIVATE : FUTEX_SHARED);
return 0;
}
sem.newsem.nwaiters = 0;
/* This always is a shared semaphore. */
- sem.newsem.private = LLL_SHARED;
+ sem.newsem.private = FUTEX_SHARED;
/* Initialize the remaining bytes as well. */
memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0',
#include <lowlevellock.h>
#include <internaltypes.h>
#include <semaphore.h>
+#include <futex-internal.h>
#include <shlib-compat.h>
-/* Wrapper for lll_futex_wake, with error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
- int res = lll_futex_wake (futex, processes_to_wake, private);
- /* No error. Ignore the number of woken processes. */
- if (res >= 0)
- return;
- switch (res)
- {
- case -EFAULT: /* Could have happened due to memory reuse. */
- case -EINVAL: /* Could be either due to incorrect alignment (a bug in
- glibc or in the application) or due to memory being
- reused for a PI futex. We cannot distinguish between the
- two causes, and one of them is correct use, so we do not
- act in this case. */
- return;
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
/* See sem_wait for an explanation of the algorithm. */
int
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#include <lowlevellock.h> /* lll_futex* used by the old code. */
#include "sem_waitcommon.c"
int
#include <errno.h>
#include <sysdep.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
#include <internaltypes.h>
#include <semaphore.h>
#include <sys/time.h>
#include <shlib-compat.h>
#include <atomic.h>
-/* Wrapper for lll_futex_wait with absolute timeout and error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline int
-futex_abstimed_wait (unsigned int* futex, unsigned int expected,
- const struct timespec* abstime, int private, bool cancel)
-{
- int err, oldtype;
- if (abstime == NULL)
- {
- if (cancel)
- oldtype = __pthread_enable_asynccancel ();
- err = lll_futex_wait (futex, expected, private);
- if (cancel)
- __pthread_disable_asynccancel (oldtype);
- }
- else
- {
- struct timeval tv;
- struct timespec rt;
- int sec, nsec;
-
- /* Get the current time. */
- __gettimeofday (&tv, NULL);
-
- /* Compute relative timeout. */
- sec = abstime->tv_sec - tv.tv_sec;
- nsec = abstime->tv_nsec - tv.tv_usec * 1000;
- if (nsec < 0)
- {
- nsec += 1000000000;
- --sec;
- }
-
- /* Already timed out? */
- if (sec < 0)
- return ETIMEDOUT;
-
- /* Do wait. */
- rt.tv_sec = sec;
- rt.tv_nsec = nsec;
- if (cancel)
- oldtype = __pthread_enable_asynccancel ();
- err = lll_futex_timed_wait (futex, expected, &rt, private);
- if (cancel)
- __pthread_disable_asynccancel (oldtype);
- }
- switch (err)
- {
- case 0:
- case -EAGAIN:
- case -EINTR:
- case -ETIMEDOUT:
- return -err;
-
- case -EFAULT: /* Must have been caused by a glibc or application bug. */
- case -EINVAL: /* Either due to wrong alignment or due to the timeout not
- being normalized. Must have been caused by a glibc or
- application bug. */
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
-/* Wrapper for lll_futex_wake, with error checking.
- TODO Remove when cleaning up the futex API throughout glibc. */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
- int res = lll_futex_wake (futex, processes_to_wake, private);
- /* No error. Ignore the number of woken processes. */
- if (res >= 0)
- return;
- switch (res)
- {
- case -EFAULT: /* Could have happened due to memory reuse. */
- case -EINVAL: /* Could be either due to incorrect alignment (a bug in
- glibc or in the application) or due to memory being
- reused for a PI futex. We cannot distinguish between the
- two causes, and one of them is correct use, so we do not
- act in this case. */
- return;
- case -ENOSYS: /* Must have been caused by a glibc bug. */
- /* No other errors are documented at this time. */
- default:
- abort ();
- }
-}
-
-
-/* Set this to true if you assume that, in contrast to current Linux futex
- documentation, lll_futex_wake can return -EINTR only if interrupted by a
- signal, not spuriously due to some other reason.
- TODO Discuss EINTR conditions with the Linux kernel community. For
- now, we set this to true to not change behavior of semaphores compared
- to previous glibc builds. */
-static const int sem_assume_only_signals_cause_futex_EINTR = 1;
static void
__sem_wait_32_finish (struct new_sem *sem);
{
int err;
- err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime,
- sem->private, true);
+ err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK,
+ abstime, sem->private);
return err;
}
__sparc32_atomic_do_unlock24(&sem->pad);
err = do_futex_wait(sem, abstime);
- if (err == ETIMEDOUT ||
- (err == EINTR && sem_assume_only_signals_cause_futex_EINTR))
+ if (err == ETIMEDOUT || err == EINTR)
{
__set_errno (err);
err = -1;
--- /dev/null
+/* futex operations for glibc-internal use. Linux version.
+ Copyright (C) 2014-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef FUTEX_INTERNAL_H
+#define FUTEX_INTERNAL_H
+
+#include <sysdeps/nptl/futex-internal.h>
+#include <errno.h>
+#include <lowlevellock-futex.h>
+#include <nptl/pthreadP.h>
+
+/* See sysdeps/nptl/futex-internal.h for documentation; this file only
+ contains Linux-specific comments.
+
+ The Linux kernel treats provides absolute timeouts based on the
+ CLOCK_REALTIME clock and relative timeouts measured against the
+ CLOCK_MONOTONIC clock.
+
+ We expect a Linux kernel version of 2.6.22 or more recent (since this
+ version, EINTR is not returned on spurious wake-ups anymore). */
+
+/* FUTEX_SHARED is always supported by the Linux kernel. */
+static __always_inline int
+futex_supports_pshared (int pshared)
+{
+ if (__glibc_likely (pshared == PTHREAD_PROCESS_PRIVATE))
+ return 0;
+ else if (pshared == PTHREAD_PROCESS_SHARED)
+ return 0;
+ else
+ return EINVAL;
+}
+
+/* The Linux kernel supports relative timeouts measured against the
+ CLOCK_MONOTONIC clock. */
+static __always_inline bool
+futex_supports_exact_relative_timeouts (void)
+{
+ return true;
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_wait (unsigned int *futex_word, unsigned int expected, int private)
+{
+ int err = lll_futex_timed_wait (futex_word, expected, NULL, private);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ return -err;
+
+ case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_wait_cancelable (unsigned int *futex_word, unsigned int expected,
+ int private)
+{
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = lll_futex_timed_wait (futex_word, expected, NULL, private);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ return -err;
+
+ case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_reltimed_wait (unsigned int *futex_word, unsigned int expected,
+ const struct timespec *reltime, int private)
+{
+ int err = lll_futex_timed_wait (futex_word, expected, reltime, private);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_reltimed_wait_cancelable (unsigned int *futex_word,
+ unsigned int expected,
+ const struct timespec *reltime, int private)
+{
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = lll_futex_timed_wait (futex_word, expected, reltime, private);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_abstimed_wait (unsigned int *futex_word, unsigned int expected,
+ const struct timespec *abstime, int private)
+{
+ /* Work around the fact that the kernel rejects negative timeout values
+ despite them being valid. */
+ if (__glibc_unlikely ((abstime != NULL) && (abstime->tv_sec < 0)))
+ return ETIMEDOUT;
+ int err = lll_futex_timed_wait_bitset (futex_word, expected, abstime,
+ FUTEX_CLOCK_REALTIME, private);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline int
+futex_abstimed_wait_cancelable (unsigned int *futex_word,
+ unsigned int expected,
+ const struct timespec *abstime, int private)
+{
+ /* Work around the fact that the kernel rejects negative timeout values
+ despite them being valid. */
+ if (__glibc_unlikely ((abstime != NULL) && (abstime->tv_sec < 0)))
+ return ETIMEDOUT;
+ int oldtype;
+ oldtype = __pthread_enable_asynccancel ();
+ int err = lll_futex_timed_wait_bitset (futex_word, expected, abstime,
+ FUTEX_CLOCK_REALTIME, private);
+ __pthread_disable_asynccancel (oldtype);
+ switch (err)
+ {
+ case 0:
+ case -EAGAIN:
+ case -EINTR:
+ case -ETIMEDOUT:
+ return -err;
+
+ case -EFAULT: /* Must have been caused by a glibc or application bug. */
+ case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+ being normalized. Must have been caused by a glibc or
+ application bug. */
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+/* See sysdeps/nptl/futex-internal.h for details. */
+static __always_inline void
+futex_wake (unsigned int *futex_word, int processes_to_wake, int private)
+{
+ int res = lll_futex_wake (futex_word, processes_to_wake, private);
+ /* No error. Ignore the number of woken processes. */
+ if (res >= 0)
+ return;
+ switch (res)
+ {
+ case -EFAULT: /* Could have happened due to memory reuse. */
+ case -EINVAL: /* Could be either due to incorrect alignment (a bug in
+ glibc or in the application) or due to memory being
+ reused for a PI futex. We cannot distinguish between the
+ two causes, and one of them is correct use, so we do not
+ act in this case. */
+ return;
+ case -ENOSYS: /* Must have been caused by a glibc bug. */
+ /* No other errors are documented at this time. */
+ default:
+ futex_fatal_error ();
+ }
+}
+
+#endif /* futex-internal.h */
projects / thirdparty / glibc.git / commitdiff
