crypto/threads_lock_contention: factor out obtaining the stack traces data pointer

It also drops the premature initalisation of it in
ossl_init_rwlock_contention_data(), deferring it to on-demand one
in ossl_rwlock_{rd,wr}lock(), which seems to shave some of the incurred
overhead:

    [Before]
    ~/dev/perftools/source$ ./evp_fetch 100
    Average time per fetch call: 16.944004us
    ~/dev/perftools/source$ ./evp_fetch 200
    Average time per fetch call: 26.325767us

    [After]
    ~/dev/perftools/source$ ./evp_fetch 100
    Average time per fetch call: 13.079795us
    ~/dev/perftools/source$ ./evp_fetch 200
    Average time per fetch call: 23.420235us

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Saša Nedvědický <sashan@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/27983)

diff --git a/crypto/threads_pthread.c b/crypto/threads_pthread.c
index e419f210832a4aa1eb3fbd2b602b5124539c6e49..cc611d2ad03be199e7a213b27a10691c4054ea64 100644 (file)
--- a/crypto/threads_pthread.c
+++ b/crypto/threads_pthread.c
@@ -38,6 +38,7 @@
 #include "internal/threads_common.h"
 #include "internal/rcu.h"
 #ifdef REPORT_RWLOCK_CONTENTION
+# include <stdbool.h>
 # include "internal/time.h"
 #endif
 #include "rcu_internal.h"
@@ -656,6 +657,18 @@ static void init_contention_fp_once(void)
     return;
 }
 
+static struct stack_traces *get_stack_traces(bool init)
+{
+    struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
+
+    if (!traces && init) {
+        traces = OPENSSL_zalloc(sizeof(*traces));
+        CRYPTO_THREAD_set_local(&thread_contention_data, traces);
+    }
+
+    return traces;
+}
+
 static void print_stack_traces(struct stack_traces *traces, FILE *fptr)
 {
     unsigned int j;
@@ -684,15 +697,6 @@ static ossl_inline void ossl_init_rwlock_contention_data(void)
 {
     CRYPTO_THREAD_run_once(&init_contention_fp, init_contention_fp_once);
     __atomic_add_fetch(&rwlock_count, 1, __ATOMIC_ACQ_REL);
-    {
-        struct stack_traces *thread_stack_info;
-
-        thread_stack_info = CRYPTO_THREAD_get_local(&thread_contention_data);
-        if (thread_stack_info == NULL) {
-            thread_stack_info = OPENSSL_zalloc(sizeof(struct stack_traces));
-            CRYPTO_THREAD_set_local(&thread_contention_data, thread_stack_info);
-        }
-    }
 }
 
 static ossl_inline void ossl_free_rwlock_contention_data(void)
@@ -712,14 +716,10 @@ static ossl_inline void ossl_free_rwlock_contention_data(void)
 
 static ossl_inline int ossl_rwlock_rdlock(pthread_rwlock_t *lock)
 {
-    struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
+    struct stack_traces *traces = get_stack_traces(true);
 
-    if (ossl_unlikely(traces == NULL)) {
-        traces = OPENSSL_zalloc(sizeof(struct stack_traces));
-        CRYPTO_THREAD_set_local(&thread_contention_data, traces);
-        if (ossl_unlikely(traces == NULL))
-            return ENOMEM;
-    }
+    if (ossl_unlikely(traces == NULL))
+        return ENOMEM;
 
     traces->lock_depth++;
     if (pthread_rwlock_tryrdlock(lock)) {
@@ -752,14 +752,10 @@ static ossl_inline int ossl_rwlock_rdlock(pthread_rwlock_t *lock)
 
 static ossl_inline int ossl_rwlock_wrlock(pthread_rwlock_t *lock)
 {
-    struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
+    struct stack_traces *traces = get_stack_traces(true);
 
-    if (ossl_unlikely(traces == NULL)) {
-        traces = OPENSSL_zalloc(sizeof(struct stack_traces));
-        CRYPTO_THREAD_set_local(&thread_contention_data, traces);
-        if (ossl_unlikely(traces == NULL))
-            return ENOMEM;
-    }
+    if (ossl_unlikely(traces == NULL))
+        return ENOMEM;
 
     traces->lock_depth++;
     if (pthread_rwlock_trywrlock(lock)) {
@@ -799,7 +795,7 @@ static ossl_inline int ossl_rwlock_unlock(pthread_rwlock_t *lock)
         return ret;
 
     {
-        struct stack_traces *traces = CRYPTO_THREAD_get_local(&thread_contention_data);
+        struct stack_traces *traces = get_stack_traces(false);
 
         if (contention_fp != NULL && traces != NULL) {
             traces->lock_depth--;