Use __thread on Linux (and ESX) in vthreadBase.c.

This change makes use of __thread on Linux for storing the vthread
base pointer and the thread ID.  We still also store the value with
pthreads_setspecific so that we have consistent behavior on thread
destruction (using the pthreads destructor).  But this allows the
VThread_CurID path to skip loading the pthread key, skip checking if
the key is initialized, and skip the call into pthread_getspecific.

(With VThread_CurID taking its own fast path, it's not clear there's
much benefit at all from optimizing the path for the base pointer, but
I did so for consistency's sake.)

I tried using my USB bandwidth test to measure the performance impact
of this on a Linux host.  Unfortunately I found the test to be much
more noisy under Linux.  The results claimed 1.6% faster overall, but
I have almost no faith in that number.

Still, having inspected the new code and looked at the contents of
pthread_getspecific, I think it's safe to claim that this should be
better overall even if its difficult to quantify how much.  And though
it makes the code slightly more crufty, such ugliness is confined to
two functions.

Signed-off-by: Dmitry Torokhov <dtor@vmware.com>

diff --git a/open-vm-tools/lib/misc/vthreadBase.c b/open-vm-tools/lib/misc/vthreadBase.c
index 6b0dac06b3b2b31bef88e159eb7a9aa63a8324c7..54c13b59137ca59b3d60dbce51c68230063ff52a 100644 (file)
--- a/open-vm-tools/lib/misc/vthreadBase.c
+++ b/open-vm-tools/lib/misc/vthreadBase.c
@@ -61,10 +61,20 @@
  *      threads.  If lib/thread is used on top of this library, the lib/thread
  *      NoID function may introduce a smaller limit.
  *
- *      On Windows and Mac, native OS TLS support is available.
- *      On Linux, require pthreads. (Generally this implies -pthread and _REENTRANT)
+ *      On Linux we make use of a combination of __thread and pthread support.
+ *      On Mac our only option is to use pthreads.
+ *      On Windows we could use the compiler supported thread locals but
+ *      that has issues when dynamically loaded from a library, so instead
+ *      we use the safer Tls* functions.
  */
 
+#if defined __linux__
+#  include <features.h>  /* for __GLIBC_PREREQ */
+#  if __GLIBC_PREREQ(2, 3)
+#    define HAVE_TLS
+#  endif
+#endif
+
 #if defined _WIN32
 #  include <windows.h>
 #else
@@ -118,6 +128,11 @@ typedef pthread_key_t VThreadBaseKeyType;
 #endif
 #endif
 
+#ifdef HAVE_TLS
+static __thread VThreadBaseData *tlsBaseCache = NULL;
+static __thread VThreadID tlsIDCache = VTHREAD_INVALID_ID;
+#endif
+
 static void VThreadBaseInit(void);
 static void VThreadBaseSimpleNoID(void);
 static void VThreadBaseSimpleFreeID(void *tlsData);
@@ -280,6 +295,11 @@ VThreadBaseAreKeysInited(void)
  *      read before it is initialized we return -1 instead of 0 (since
  *      0 is used as a real thread id).
  *
+ *      Note that we use store the thread local value using pthreads
+ *      even when HAVE_TLS is defined.  This way we continue to use
+ *      the same pthread destructor path for cleanup as we would
+ *      without HAVE_TLS.
+ *
  *-----------------------------------------------------------------------------
  */
 
@@ -287,6 +307,7 @@ static INLINE Bool
 VThreadBaseSetLocal(VThreadLocal local, void *value)
 {
    VThreadBaseKeyType key;
+   void *adjustedValue = value;
    Bool success;
    ASSERT(VThreadBaseAreKeysInited());
    if (local == VTHREAD_LOCAL_BASE) {
@@ -294,14 +315,24 @@ VThreadBaseSetLocal(VThreadLocal local, void *value)
    } else {
       ASSERT(local == VTHREAD_LOCAL_ID);
       /* VThreadGetLocal compensates for this, lets the default be -1.  */
-      value = (void*)((uintptr_t)value + 1);
+      adjustedValue = (void*)((uintptr_t)adjustedValue + 1);
       key = Atomic_Read(&vthreadBaseGlobals.threadIDKey);
    }
    ASSERT(key != VTHREADBASE_INVALID_KEY);
 #if defined _WIN32
-   success = TlsSetValue(key, value);
+   success = TlsSetValue(key, adjustedValue);
 #else
-   success = pthread_setspecific(key, value) == 0;
+   success = pthread_setspecific(key, adjustedValue) == 0;
+#endif
+#ifdef HAVE_TLS
+   if (success) {
+      if (local == VTHREAD_LOCAL_BASE) {
+         tlsBaseCache = value;
+      } else {
+         ASSERT(local == VTHREAD_LOCAL_ID);
+         tlsIDCache = (VThreadID)(uintptr_t)value;
+      }
+   }
 #endif
    return success;
 }
@@ -322,11 +353,17 @@ VThreadBaseSetLocal(VThreadLocal local, void *value)
 static INLINE void *
 VThreadBaseGetLocal(VThreadLocal local) 
 {
+   void *result;
+#ifdef HAVE_TLS
+   if (local == VTHREAD_LOCAL_BASE) {
+      result = tlsBaseCache;
+   } else {
+      ASSERT(local == VTHREAD_LOCAL_ID);
+      result = (void*)(uintptr_t)tlsIDCache;
+   }
+#else 
    VThreadBaseKeyType key;
    Atomic_Int *keyPtr;
-   void *result;
-
-   ASSERT(local == VTHREAD_LOCAL_BASE || local == VTHREAD_LOCAL_ID);
 
    keyPtr = local == VTHREAD_LOCAL_BASE ? &vthreadBaseGlobals.baseKey :
                                           &vthreadBaseGlobals.threadIDKey;
@@ -342,10 +379,10 @@ VThreadBaseGetLocal(VThreadLocal local)
 #else
    result = pthread_getspecific(key);
 #endif
-
    if (local == VTHREAD_LOCAL_ID) {
       result = (void*)((uintptr_t)result - 1); /* See VThreadBaseSetLocal. */
    }
+#endif
 
    return result;
 }
@@ -669,7 +706,8 @@ VThreadBaseSafeDeleteTLS(void *tlsData)
          }
          (*vthreadBaseGlobals.freeIDFunc)(data);
 
-         success = VThreadBaseSetBase(NULL) && VThreadBaseSetID(0);
+         success = VThreadBaseSetBase(NULL) &&
+                   VThreadBaseSetID(VTHREAD_INVALID_ID);
          ASSERT_NOT_IMPLEMENTED(success);
       }
       Atomic_Dec(&vthreadBaseGlobals.numThreads);