openssl: guard against OOM on context creation

EVP_MD_CTX_create will allocate memory for the context and returns
NULL in case the allocation fails. Make sure to catch any allocation
failures and exit early if so.

In passing, also move to EVP_DigestInit rather than EVP_DigestInit_ex
as the latter is intended for ENGINE selection which we don't do.

Closes #6224
Reviewed-by: Daniel Stenberg <daniel@haxx.se>
Reviewed-by: Emil Engler <me@emilengler.com>

diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index 1803ed34c124accfb1bb4480574ff1a4eb5b6cce..fc0c4e15fd095c44eca8e66be3f8d0c96bf8e8bd 100644 (file)
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -4364,7 +4364,9 @@ static CURLcode Curl_ossl_md5sum(unsigned char *tmp, /* input */
   (void) unused;
 
   mdctx = EVP_MD_CTX_create();
-  EVP_DigestInit_ex(mdctx, EVP_md5(), NULL);
+  if(!mdctx)
+    return CURLE_OUT_OF_MEMORY;
+  EVP_DigestInit(mdctx, EVP_md5());
   EVP_DigestUpdate(mdctx, tmp, tmplen);
   EVP_DigestFinal_ex(mdctx, md5sum, &len);
   EVP_MD_CTX_destroy(mdctx);
@@ -4382,7 +4384,9 @@ static CURLcode Curl_ossl_sha256sum(const unsigned char *tmp, /* input */
   (void) unused;
 
   mdctx = EVP_MD_CTX_create();
-  EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL);
+  if(!mdctx)
+    return CURLE_OUT_OF_MEMORY;
+  EVP_DigestInit(mdctx, EVP_sha256());
   EVP_DigestUpdate(mdctx, tmp, tmplen);
   EVP_DigestFinal_ex(mdctx, sha256sum, &len);
   EVP_MD_CTX_destroy(mdctx);