EVP_MD performance fix (refcount cache contention)

Partial fix for #17064. Avoid excessive writes to the cache line
containing the refcount for an EVP_MD object to avoid extreme
cache contention when using a single EVP_MD at high frequency on
multiple threads. This changes performance in 3.0 from being double
that of 1.1 to only slightly higher than that of 1.1.

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17857)

diff --git a/crypto/evp/digest.c b/crypto/evp/digest.c
index db2eed63551ac87c6d5f9c870d6bb8d50131f954..cd930ebd7a8eecdaa8e118edd0aacf758da4253f 100644 (file)
--- a/crypto/evp/digest.c
+++ b/crypto/evp/digest.c
@@ -40,7 +40,7 @@ static void cleanup_old_md_data(EVP_MD_CTX *ctx, int force)
     }
 }
 
-void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force)
+void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force, int keep_fetched)
 {
     if (ctx->algctx != NULL) {
         if (ctx->digest != NULL && ctx->digest->freectx != NULL)
@@ -65,13 +65,14 @@ void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force)
 #endif
 
     /* Non legacy code, this has to be later than the ctx->digest cleaning */
-    EVP_MD_free(ctx->fetched_digest);
-    ctx->fetched_digest = NULL;
-    ctx->reqdigest = NULL;
+    if (!keep_fetched) {
+        EVP_MD_free(ctx->fetched_digest);
+        ctx->fetched_digest = NULL;
+        ctx->reqdigest = NULL;
+    }
 }
 
-/* This call frees resources associated with the context */
-int EVP_MD_CTX_reset(EVP_MD_CTX *ctx)
+static int evp_md_ctx_reset_ex(EVP_MD_CTX *ctx, int keep_fetched)
 {
     if (ctx == NULL)
         return 1;
@@ -87,12 +88,19 @@ int EVP_MD_CTX_reset(EVP_MD_CTX *ctx)
     }
 #endif
 
-    evp_md_ctx_clear_digest(ctx, 0);
-    OPENSSL_cleanse(ctx, sizeof(*ctx));
+    evp_md_ctx_clear_digest(ctx, 0, keep_fetched);
+    if (!keep_fetched)
+        OPENSSL_cleanse(ctx, sizeof(*ctx));
 
     return 1;
 }
 
+/* This call frees resources associated with the context */
+int EVP_MD_CTX_reset(EVP_MD_CTX *ctx)
+{
+    return evp_md_ctx_reset_ex(ctx, 0);
+}
+
 #ifndef FIPS_MODULE
 EVP_MD_CTX *evp_md_ctx_new_ex(EVP_PKEY *pkey, const ASN1_OCTET_STRING *id,
                               OSSL_LIB_CTX *libctx, const char *propq)
@@ -524,6 +532,7 @@ int EVP_MD_CTX_copy(EVP_MD_CTX *out, const EVP_MD_CTX *in)
 
 int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in)
 {
+    int digest_change = 0;
     unsigned char *tmp_buf;
 
     if (in == NULL) {
@@ -549,15 +558,16 @@ int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in)
         return 0;
     }
 
-    EVP_MD_CTX_reset(out);
-    if (out->fetched_digest != NULL)
+    evp_md_ctx_reset_ex(out, 1);
+    digest_change = (out->fetched_digest != in->fetched_digest);
+    if (digest_change && out->fetched_digest != NULL)
         EVP_MD_free(out->fetched_digest);
     *out = *in;
     /* NULL out pointers in case of error */
     out->pctx = NULL;
     out->algctx = NULL;
 
-    if (in->fetched_digest != NULL)
+    if (digest_change && in->fetched_digest != NULL)
         EVP_MD_up_ref(in->fetched_digest);
 
     if (in->algctx != NULL) {

diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index 0993de0937722cb520b8b4588bdc63566beb3faa..371bca500143732701e05d0c623594fd93ba0b78 100644 (file)
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -231,7 +231,7 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
              * We're about to get a new digest so clear anything associated with
              * an old digest.
              */
-            evp_md_ctx_clear_digest(ctx, 1);
+            evp_md_ctx_clear_digest(ctx, 1, 0);
 
             /* legacy code support for engines */
             ERR_set_mark();

diff --git a/include/crypto/evp.h b/include/crypto/evp.h
index c5d3a930f749772aaab6e837d50326f8d2c5440a..6fb05309dc9fbb54ade6ed0a9f651a881c70739b 100644 (file)
--- a/include/crypto/evp.h
+++ b/include/crypto/evp.h
@@ -900,7 +900,7 @@ int evp_set_default_properties_int(OSSL_LIB_CTX *libctx, const char *propq,
                                    int loadconfig, int mirrored);
 char *evp_get_global_properties_str(OSSL_LIB_CTX *libctx, int loadconfig);
 
-void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force);
+void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force, int keep_digest);
 
 /* Three possible states: */
 # define EVP_PKEY_STATE_UNKNOWN         0