Factor out the lock contention reporting facility implementation

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Saša Nedvědický <sashan@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/27983)

diff --git a/crypto/threads_pthread.c b/crypto/threads_pthread.c
index df7954f84cc734182b4ca648740c88be32eb11bc..338c1fb18567b67638fa4617cc27209d3e09f5a1 100644 (file)
--- a/crypto/threads_pthread.c
+++ b/crypto/threads_pthread.c
@@ -630,6 +630,7 @@ struct stack_info {
 };
 
 #  define STACKS_COUNT 32
+#  define BT_BUF_SIZE 1024
 struct stack_traces {
     int lock_depth;
     size_t idx;
@@ -653,68 +654,7 @@ static void init_contention_fp_once(void)
     CRYPTO_THREAD_init_local(&thread_contention_data, destroy_contention_data);
     return;
 }
-# endif
-
-CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void)
-{
-# ifdef USE_RWLOCK
-    CRYPTO_RWLOCK *lock;
 
-#  ifdef REPORT_RWLOCK_CONTENTION
-    CRYPTO_THREAD_run_once(&init_contention_fp, init_contention_fp_once);
-    __atomic_add_fetch(&rwlock_count, 1, __ATOMIC_ACQ_REL);
-    {
-        struct stack_info *thread_stack_info;
-
-        thread_stack_info = CRYPTO_THREAD_get_local(&thread_contention_data);
-        if (thread_stack_info == NULL) {
-            thread_stack_info = OPENSSL_zalloc(sizeof(struct stack_traces));
-            CRYPTO_THREAD_set_local(&thread_contention_data, thread_stack_info);
-        }
-    }
-#  endif
-
-    if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL)
-        /* Don't set error, to avoid recursion blowup. */
-        return NULL;
-
-    if (pthread_rwlock_init(lock, NULL) != 0) {
-        OPENSSL_free(lock);
-        return NULL;
-    }
-# else
-    pthread_mutexattr_t attr;
-    CRYPTO_RWLOCK *lock;
-
-    if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL)
-        /* Don't set error, to avoid recursion blowup. */
-        return NULL;
-
-    /*
-     * We don't use recursive mutexes, but try to catch errors if we do.
-     */
-    pthread_mutexattr_init(&attr);
-#  if !defined (__TANDEM) && !defined (_SPT_MODEL_)
-#   if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK)
-    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
-#   endif
-#  else
-    /* The SPT Thread Library does not define MUTEX attributes. */
-#  endif
-
-    if (pthread_mutex_init(lock, &attr) != 0) {
-        pthread_mutexattr_destroy(&attr);
-        OPENSSL_free(lock);
-        return NULL;
-    }
-
-    pthread_mutexattr_destroy(&attr);
-# endif
-
-    return lock;
-}
-
-# ifdef REPORT_RWLOCK_CONTENTION
 static void print_stack_traces(struct stack_traces *traces, FILE *fptr)
 {
     unsigned int j;
@@ -738,32 +678,59 @@ static void print_stack_traces(struct stack_traces *traces, FILE *fptr)
     }
     pthread_mutex_unlock(&log_lock);
 }
-# endif
 
-# define BT_BUF_SIZE 1024
+static ossl_inline void ossl_init_rwlock_contention_data(void)
+{
+    CRYPTO_THREAD_run_once(&init_contention_fp, init_contention_fp_once);
+    __atomic_add_fetch(&rwlock_count, 1, __ATOMIC_ACQ_REL);
+    {
+        struct stack_info *thread_stack_info;
 
-__owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock)
+        thread_stack_info = CRYPTO_THREAD_get_local(&thread_contention_data);
+        if (thread_stack_info == NULL) {
+            thread_stack_info = OPENSSL_zalloc(sizeof(struct stack_traces));
+            CRYPTO_THREAD_set_local(&thread_contention_data, thread_stack_info);
+        }
+    }
+}
+
+static ossl_inline void ossl_free_rwlock_contention_data(void)
+{
+    /*
+     * Note: It's possible here that OpenSSL may allocate a lock and immediately
+     * free it, in which case we would erroneously close the contention log
+     * prior to the library going on to do more real work.  In practice
+     * that never happens though, and since this is a debug facility
+     * we don't worry about that here.
+     */
+    if (__atomic_add_fetch(&rwlock_count, -1, __ATOMIC_ACQ_REL) == 0) {
+        fclose(contention_fp);
+        contention_fp = NULL;
+    }
+}
+
+static ossl_inline int ossl_rwlock_rdlock(pthread_rwlock_t *lock)
 {
-# ifdef USE_RWLOCK
-#  ifdef REPORT_RWLOCK_CONTENTION
     struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
 
     if (ossl_unlikely(traces == NULL)) {
         traces = OPENSSL_zalloc(sizeof(struct stack_traces));
         CRYPTO_THREAD_set_local(&thread_contention_data, traces);
         if (ossl_unlikely(traces == NULL))
-            return 0;
+            return ENOMEM;
     }
 
     traces->lock_depth++;
     if (pthread_rwlock_tryrdlock(lock)) {
         void *buffer[BT_BUF_SIZE];
         OSSL_TIME start, end;
+        int ret;
 
         start = ossl_time_now();
-        if (!ossl_assert(pthread_rwlock_rdlock(lock) == 0)) {
+        ret = pthread_rwlock_rdlock(lock);
+        if (ret) {
             traces->lock_depth--;
-            return 0;
+            return ret;
         }
         end = ossl_time_now();
         traces->stacks[traces->idx].duration = ossl_time_subtract(end, start);
@@ -779,42 +746,32 @@ __owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock)
             print_stack_traces(traces, contention_fp);
         }
     }
-#  else
-    if (!ossl_assert(pthread_rwlock_rdlock(lock) == 0))
-        return 0;
-#  endif
-# else
-    if (pthread_mutex_lock(lock) != 0) {
-        assert(errno != EDEADLK && errno != EBUSY);
-        return 0;
-    }
-# endif
 
-    return 1;
+    return 0;
 }
 
-__owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock)
+static ossl_inline int ossl_rwlock_wrlock(pthread_rwlock_t *lock)
 {
-# ifdef USE_RWLOCK
-#  ifdef REPORT_RWLOCK_CONTENTION
     struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
 
     if (ossl_unlikely(traces == NULL)) {
         traces = OPENSSL_zalloc(sizeof(struct stack_traces));
         CRYPTO_THREAD_set_local(&thread_contention_data, traces);
         if (ossl_unlikely(traces == NULL))
-            return 0;
+            return ENOMEM;
     }
 
     traces->lock_depth++;
     if (pthread_rwlock_trywrlock(lock)) {
         void *buffer[BT_BUF_SIZE];
         OSSL_TIME start, end;
+        int ret;
 
         start = ossl_time_now();
-        if (!ossl_assert(pthread_rwlock_wrlock(lock) == 0)) {
+        ret = pthread_rwlock_wrlock(lock);
+        if (ret) {
             traces->lock_depth--;
-            return 0;
+            return ret;
         }
         end = ossl_time_now();
         traces->stacks[traces->idx].nptrs = backtrace(buffer, BT_BUF_SIZE);
@@ -829,10 +786,110 @@ __owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock)
             print_stack_traces(traces, contention_fp);
         }
     }
+
+    return 0;
+}
+
+static ossl_inline int ossl_rwlock_unlock(pthread_rwlock_t *lock)
+{
+    int ret;
+
+    ret = pthread_rwlock_unlock(lock);
+    if (ret)
+        return ret;
+
+    {
+        struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
+
+        if (contention_fp != NULL && traces != NULL) {
+            traces->lock_depth--;
+            assert(traces->lock_depth >= 0);
+            if (traces->lock_depth == 0)
+                print_stack_traces(traces, contention_fp);
+        }
+    }
+
+    return 0;
+}
+
+# else /* !REPORT_RWLOCK_CONTENTION */
+
+static ossl_inline void ossl_init_rwlock_contention_data(void)
+{
+}
+
+static ossl_inline void ossl_free_rwlock_contention_data(void)
+{
+}
+
+static ossl_inline int ossl_rwlock_rdlock(pthread_rwlock_t *rwlock)
+{
+    return pthread_rwlock_rdlock(rwlock);
+}
+
+static ossl_inline int ossl_rwlock_wrlock(pthread_rwlock_t *rwlock)
+{
+    return pthread_rwlock_wrlock(rwlock);
+}
+
+static ossl_inline int ossl_rwlock_unlock(pthread_rwlock_t *rwlock)
+{
+    return pthread_rwlock_unlock(rwlock);
+}
+# endif /* REPORT_RWLOCK_CONTENTION */
+
+CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void)
+{
+# ifdef USE_RWLOCK
+    CRYPTO_RWLOCK *lock;
+
+    ossl_init_rwlock_contention_data();
+
+    if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL)
+        /* Don't set error, to avoid recursion blowup. */
+        return NULL;
+
+    if (pthread_rwlock_init(lock, NULL) != 0) {
+        OPENSSL_free(lock);
+        return NULL;
+    }
+# else
+    pthread_mutexattr_t attr;
+    CRYPTO_RWLOCK *lock;
+
+    if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL)
+        /* Don't set error, to avoid recursion blowup. */
+        return NULL;
+
+    /*
+     * We don't use recursive mutexes, but try to catch errors if we do.
+     */
+    pthread_mutexattr_init(&attr);
+#  if !defined (__TANDEM) && !defined (_SPT_MODEL_)
+#   if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK)
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
+#   endif
 #  else
-    if (!ossl_assert(pthread_rwlock_wrlock(lock) == 0))
-        return 0;
+    /* The SPT Thread Library does not define MUTEX attributes. */
 #  endif
+
+    if (pthread_mutex_init(lock, &attr) != 0) {
+        pthread_mutexattr_destroy(&attr);
+        OPENSSL_free(lock);
+        return NULL;
+    }
+
+    pthread_mutexattr_destroy(&attr);
+# endif
+
+    return lock;
+}
+
+__owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock)
+{
+# ifdef USE_RWLOCK
+    if (!ossl_assert(ossl_rwlock_rdlock(lock) == 0))
+        return 0;
 # else
     if (pthread_mutex_lock(lock) != 0) {
         assert(errno != EDEADLK && errno != EBUSY);
@@ -843,23 +900,26 @@ __owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock)
     return 1;
 }
 
-int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock)
+__owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock)
 {
 # ifdef USE_RWLOCK
-    if (pthread_rwlock_unlock(lock) != 0)
+    if (!ossl_assert(ossl_rwlock_wrlock(lock) == 0))
+        return 0;
+# else
+    if (pthread_mutex_lock(lock) != 0) {
+        assert(errno != EDEADLK && errno != EBUSY);
         return 0;
-#  ifdef REPORT_RWLOCK_CONTENTION
-    {
-        struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
-
-        if (contention_fp != NULL && traces != NULL) {
-            traces->lock_depth--;
-            assert(traces->lock_depth >= 0);
-            if (traces->lock_depth == 0)
-                print_stack_traces(traces, contention_fp);
-        }
     }
-#  endif
+# endif
+
+    return 1;
+}
+
+int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock)
+{
+# ifdef USE_RWLOCK
+    if (ossl_rwlock_unlock(lock) != 0)
+        return 0;
 # else
     if (pthread_mutex_unlock(lock) != 0) {
         assert(errno != EPERM);
@@ -874,22 +934,9 @@ void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock)
 {
     if (lock == NULL)
         return;
-# ifdef REPORT_RWLOCK_CONTENTION
-
-    /*
-     * Note: It's possible here that OpenSSL may allocate a lock and immediately
-     * free it, in which case we would erroneously close the contention log
-     * prior to the library going on to do more real work.  In practice
-     * that never happens though, and since this is a debug facility
-     * we don't worry about that here.
-     */
-    if (__atomic_add_fetch(&rwlock_count, -1, __ATOMIC_ACQ_REL) == 0) {
-        fclose(contention_fp);
-        contention_fp = NULL;
-    }
-# endif
 
 # ifdef USE_RWLOCK
+    ossl_free_rwlock_contention_data();
     pthread_rwlock_destroy(lock);
 # else
     pthread_mutex_destroy(lock);