Don't assume OpenSSL failures are memory errors

More recent versions of OpenSSL can fail for other reasons.  Indicate
a crypto-related error occurred rather than a memory error to aid
debugging.

(cherry picked from commit bf9b2134ceddd6c727362be894b1c95c297a0f17)

ticket: 8873
version_fixed: 1.17.2

diff --git a/src/lib/crypto/openssl/hash_provider/hash_evp.c b/src/lib/crypto/openssl/hash_provider/hash_evp.c
index 957ed8d9caaf5886fc66e0b8b6f9223fea5db412..1e0fb8fc3c8b5df71699a5c96873ca8525412972 100644 (file)
--- a/src/lib/crypto/openssl/hash_provider/hash_evp.c
+++ b/src/lib/crypto/openssl/hash_provider/hash_evp.c
@@ -58,7 +58,7 @@ hash_evp(const EVP_MD *type, const krb5_crypto_iov *data, size_t num_data,
     }
     ok = ok && EVP_DigestFinal_ex(ctx, (uint8_t *)output->data, NULL);
     EVP_MD_CTX_free(ctx);
-    return ok ? 0 : ENOMEM;
+    return ok ? 0 : KRB5_CRYPTO_INTERNAL;
 }
 
 static krb5_error_code

diff --git a/src/lib/crypto/openssl/sha256.c b/src/lib/crypto/openssl/sha256.c
index 0edd8b7ba71a72ffe7286ed2f01df4dca3e5dd82..f9dfc85393ba698439b559969d31f80b417fce40 100644 (file)
--- a/src/lib/crypto/openssl/sha256.c
+++ b/src/lib/crypto/openssl/sha256.c
@@ -48,5 +48,5 @@ k5_sha256(const krb5_data *in, size_t n, uint8_t out[K5_SHA256_HASHLEN])
         ok = ok && EVP_DigestUpdate(ctx, in[i].data, in[i].length);
     ok = ok && EVP_DigestFinal_ex(ctx, out, NULL);
     EVP_MD_CTX_free(ctx);
-    return ok ? 0 : ENOMEM;
+    return ok ? 0 : KRB5_CRYPTO_INTERNAL;
 }