nptl: Optimize trylock for high cache contention workloads (BZ #33704)

Check lock availability before acquisition to reduce cache line
bouncing.  Significantly improves trylock throughput on multi-core
systems under heavy contention.

Tested on x86_64.

Fixes BZ #33704.

Co-authored-by: Alex M Wells <alex.m.wells@intel.com>
Reviewed-by: Wilco Dijkstra  <Wilco.Dijkstra@arm.com>

diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index 94621cc25460e405632f4c29483a1e92a3d295f4..fe9ba8da671c888d686ac85d5b43e95e544068e9 100644 (file)
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -47,7 +47,8 @@ ___pthread_mutex_trylock (pthread_mutex_t *mutex)
          return 0;
        }
 
-      if (lll_trylock (mutex->__data.__lock) == 0)
+      if (atomic_load_relaxed (&(mutex->__data.__lock)) == 0
+         && lll_trylock (mutex->__data.__lock) == 0)
        {
          /* Record the ownership.  */
          mutex->__data.__owner = id;
@@ -60,7 +61,10 @@ ___pthread_mutex_trylock (pthread_mutex_t *mutex)
     case PTHREAD_MUTEX_TIMED_NP:
     case PTHREAD_MUTEX_ADAPTIVE_NP:
     case PTHREAD_MUTEX_ERRORCHECK_NP:
-      if (lll_trylock (mutex->__data.__lock) != 0)
+      /* Mutex type is already loaded, lock check overhead should
+         be minimal.  */
+      if (atomic_load_relaxed (&(mutex->__data.__lock)) != 0
+         || lll_trylock (mutex->__data.__lock) != 0)
        break;
 
       /* Record the ownership.  */