Add runtime check for __ASSUME_REQUEUE_PI (BZ# 18463)

This patch adds a runtime check for requeue priority futexes
(FUTEX_{WAIT,CMP}_REQUEUE_PI) for configurations that do not define
__ASSUME_REQUEUE_PI.

It uses the same check uses for priority lock/unlock support, where
issuing a futex operation with FUTEX_UNLOCK_PI returns if kernel has
or not internal 'futex_cmpxchg_enabled' support (which is also used
on requeue priority futexes operations).

For architectures that already have __ASSUME_REQUEUE_PI the code
flow does not change, 'use_requeue_pi' returns the previous chec
logic.  Also, if 'futex_cmpxchg_enabled' is not supported by the
kernel, the previous logic will be used (by issuing a default futex
operation).

Tested on ARM (v3.8 kernel) and x86_64.

	* nptl/pthreadP.h (prio_inherit_missing): Change name to
	__prio_inherit_missing.
	(USE_REQUEUE_PI): Remove define.
	(use_requeue_pi): New function.
	* nptl/pthread_cond_broadcast.c [__ASSUME_REQUEUE_PI]
	(__pthread_cond_broadcast): Remove ifdef.
	(USE_REQUEUE_PI): Change to use_requeue_pi.
	* nptl/pthread_cond_signal.c [__ASSUME_REQUEUE_PI]
	(__pthread_cond_signal): Remove ifdef.
	(USE_REQUEUE_PI): Change to use_requeue_pi.
	* nptl/pthread_cond_timedwait.c [__ASSUME_REQUEUE_PI]
	(__pthread_cond_timedwait): Remove ifdef.
	(USE_REQUEUE_PI): Change to use_requeue_pi.
	( nptl/pthread_cond_wait.c [__ASSUME_REQUEUE_PI]
	(__pthread_cond_wait): Remove ifdef.
	(USE_REQUEUE_PI): Change to use_requeue_pi.
	* nptl/pthread_mutex_init.c (prio_inherit_missing): Remove 'static'
	qualifier and change name to __prio_inherit_missing.

diff --git a/ChangeLog b/ChangeLog
index 9657658f615b95614bd04aa01c0cdac2207f3a33..c9b1c96fbddf26fd80234cad74dfaefaac69a0a7 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,24 @@
 2016-06-03  Adhemerval Zanella  <adhemerval.zanella@linaro.org>
 
+       * nptl/pthreadP.h (prio_inherit_missing): Change name to
+       __prio_inherit_missing.
+       (USE_REQUEUE_PI): Remove define.
+       (use_requeue_pi): New function.
+       * nptl/pthread_cond_broadcast.c [__ASSUME_REQUEUE_PI]
+       (__pthread_cond_broadcast): Remove ifdef.
+       (USE_REQUEUE_PI): Change to use_requeue_pi.
+       * nptl/pthread_cond_signal.c [__ASSUME_REQUEUE_PI]
+       (__pthread_cond_signal): Remove ifdef.
+       (USE_REQUEUE_PI): Change to use_requeue_pi.
+       * nptl/pthread_cond_timedwait.c [__ASSUME_REQUEUE_PI]
+       (__pthread_cond_timedwait): Remove ifdef.
+       (USE_REQUEUE_PI): Change to use_requeue_pi.
+       ( nptl/pthread_cond_wait.c [__ASSUME_REQUEUE_PI]
+       (__pthread_cond_wait): Remove ifdef.
+       (USE_REQUEUE_PI): Change to use_requeue_pi.
+       * nptl/pthread_mutex_init.c (prio_inherit_missing): Remove 'static'
+       qualifier and change name to __prio_inherit_missing.
+
        * nptl/nptl-init.c [__ASSUME_SET_ROBUST_LIST]
        (__set_robust_list_avail): Remove define.
        [__NR_set_robust_list] (__pthread_initialize_minimal_internal):

diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index d479a3e4e5dc280d66c81c2ae3f5bd53fb21e8c6..2b73c2974dc49a90d4789f5becf923ab2b3fd9ff 100644 (file)
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -585,17 +585,22 @@ extern void __wait_lookup_done (void) attribute_hidden;
 # define PTHREAD_STATIC_FN_REQUIRE(name) __asm (".globl " #name);
 #endif
 
+extern bool __prio_inherit_missing (void);
+
 /* Test if the mutex is suitable for the FUTEX_WAIT_REQUEUE_PI operation.  */
-#if (defined lll_futex_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
-# define USE_REQUEUE_PI(mut) \
-   ((mut) && (mut) != (void *) ~0l \
-    && (((mut)->__data.__kind \
-        & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) \
-       == PTHREAD_MUTEX_PRIO_INHERIT_NP))
-#else
-# define USE_REQUEUE_PI(mut) 0
+static inline bool
+use_requeue_pi (pthread_mutex_t *mut)
+{
+#ifndef __ASSUME_REQUEUE_PI
+  if (__prio_inherit_missing ())
+    return false;
 #endif
+  return (mut) &&
+    (mut) != (void *) ~0l &&
+    (((mut)->__data.__kind
+      & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) ==
+     PTHREAD_MUTEX_PRIO_INHERIT_NP);
+}
 
 /* Returns 0 if POL is a valid scheduling policy.  */
 static inline int

diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
index 552fd42f60148e1af54ae2a4feff74dae7003228..ef81bcd060a8e9ebe92aaa82323100a9cb361e4e 100644 (file)
--- a/nptl/pthread_cond_broadcast.c
+++ b/nptl/pthread_cond_broadcast.c
@@ -60,9 +60,8 @@ __pthread_cond_broadcast (pthread_cond_t *cond)
          || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT)
        goto wake_all;
 
-#if (defined lll_futex_cmp_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
-      if (USE_REQUEUE_PI (mut))
+#ifdef lll_futex_cmp_requeue_pi
+      if (use_requeue_pi (mut))
        {
          if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX,
                                        &mut->__data.__lock, futex_val,

diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
index b3a6d3d2a4596a51ee360f4a5ffcb9acb1129277..9c8f883bf553c7cd24b6233b56919964fd05d597 100644 (file)
--- a/nptl/pthread_cond_signal.c
+++ b/nptl/pthread_cond_signal.c
@@ -46,11 +46,10 @@ __pthread_cond_signal (pthread_cond_t *cond)
       ++cond->__data.__wakeup_seq;
       ++cond->__data.__futex;
 
-#if (defined lll_futex_cmp_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_cmp_requeue_pi
       pthread_mutex_t *mut = cond->__data.__mutex;
 
-      if (USE_REQUEUE_PI (mut)
+      if (use_requeue_pi (mut)
        /* This can only really fail with a ENOSYS, since nobody can modify
           futex while we have the cond_lock.  */
          && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,

diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
index 711a51de201d442182ef8b1c0d96ee30a57e46d6..61a651cef0823388da9f5aad2f99941e9f622894 100644 (file)
--- a/nptl/pthread_cond_timedwait.c
+++ b/nptl/pthread_cond_timedwait.c
@@ -63,8 +63,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
   int pshared = (cond->__data.__mutex == (void *) ~0l)
                ? LLL_SHARED : LLL_PRIVATE;
 
-#if (defined lll_futex_timed_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_timed_wait_requeue_pi
   int pi_flag = 0;
 #endif
 
@@ -161,8 +160,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
 
 /* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient
    to check just the former.  */
-#if (defined lll_futex_timed_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_timed_wait_requeue_pi
       /* If pi_flag remained 1 then it means that we had the lock and the mutex
         but a spurious waker raced ahead of us.  Give back the mutex before
         going into wait again.  */
@@ -171,7 +169,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
          __pthread_mutex_cond_lock_adjust (mutex);
          __pthread_mutex_unlock_usercnt (mutex, 0);
        }
-      pi_flag = USE_REQUEUE_PI (mutex);
+      pi_flag = use_requeue_pi (mutex);
 
       if (pi_flag)
        {
@@ -250,8 +248,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
   __pthread_cleanup_pop (&buffer, 0);
 
   /* Get the mutex before returning.  */
-#if (defined lll_futex_timed_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_timed_wait_requeue_pi
   if (pi_flag)
     {
       __pthread_mutex_cond_lock_adjust (mutex);

diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
index 3f62acc6bd04579ed1a9fb952e6cc50e74541c2f..f79bf67ef530bd698c7f8ede4225cde12f1f3699 100644 (file)
--- a/nptl/pthread_cond_wait.c
+++ b/nptl/pthread_cond_wait.c
@@ -86,7 +86,7 @@ __condvar_cleanup (void *arg)
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  We don't try to get the mutex if we already own it.  */
-  if (!(USE_REQUEUE_PI (cbuffer->mutex))
+  if (!(use_requeue_pi (cbuffer->mutex))
       || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK)
          != THREAD_GETMEM (THREAD_SELF, tid)))
   {
@@ -106,8 +106,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
   int pshared = (cond->__data.__mutex == (void *) ~0l)
                ? LLL_SHARED : LLL_PRIVATE;
 
-#if (defined lll_futex_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_wait_requeue_pi
   int pi_flag = 0;
 #endif
 
@@ -160,8 +159,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
-#if (defined lll_futex_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_wait_requeue_pi
       /* If pi_flag remained 1 then it means that we had the lock and the mutex
         but a spurious waker raced ahead of us.  Give back the mutex before
         going into wait again.  */
@@ -170,7 +168,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
          __pthread_mutex_cond_lock_adjust (mutex);
          __pthread_mutex_unlock_usercnt (mutex, 0);
        }
-      pi_flag = USE_REQUEUE_PI (mutex);
+      pi_flag = use_requeue_pi (mutex);
 
       if (pi_flag)
        {
@@ -221,8 +219,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
   __pthread_cleanup_pop (&buffer, 0);
 
   /* Get the mutex before returning.  Not needed for PI.  */
-#if (defined lll_futex_wait_requeue_pi \
-     && defined __ASSUME_REQUEUE_PI)
+#ifdef lll_futex_wait_requeue_pi
   if (pi_flag)
     {
       __pthread_mutex_cond_lock_adjust (mutex);

diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c
index 6aef89029e1bd7a3a856d3e626fca7545287b029..c37f623cb4926fb049142a03950ad55f5cd677c5 100644 (file)
--- a/nptl/pthread_mutex_init.c
+++ b/nptl/pthread_mutex_init.c
@@ -33,8 +33,9 @@ static const struct pthread_mutexattr default_mutexattr =
   };
 
 
-static bool
-prio_inherit_missing (void)
+bool
+attribute_hidden
+__prio_inherit_missing (void)
 {
 #ifdef __NR_futex
   static int tpi_supported;
@@ -72,7 +73,7 @@ __pthread_mutex_init (pthread_mutex_t *mutex,
       break;
 
     case PTHREAD_PRIO_INHERIT << PTHREAD_MUTEXATTR_PROTOCOL_SHIFT:
-      if (__glibc_unlikely (prio_inherit_missing ()))
+      if (__glibc_unlikely (__prio_inherit_missing ()))
        return ENOTSUP;
       break;