x509: improve error reporting

Distinguish between not being able to extract a public key versus not knowing
the key's type.

Alternative to #15921

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15944)

diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
index 1c1a5e6a27fef8cb9fd220f41a4acd43fb0eb19a..8b4e46a5895618c967efae990281a7d6521f3319 100644 (file)
--- a/crypto/x509/x509_cmp.c
+++ b/crypto/x509/x509_cmp.c
@@ -391,15 +391,12 @@ int X509_check_private_key(const X509 *x, const EVP_PKEY *k)
     int ret;
 
     xk = X509_get0_pubkey(x);
+    if (xk == NULL) {
+        ERR_raise(ERR_LIB_X509, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY);
+        return 0;
+    }
 
-    if (xk)
-        ret = EVP_PKEY_eq(xk, k);
-    else
-        ret = -2;
-
-    switch (ret) {
-    case 1:
-        break;
+    switch (ret = EVP_PKEY_eq(xk, k)) {
     case 0:
         ERR_raise(ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH);
         break;
@@ -408,10 +405,10 @@ int X509_check_private_key(const X509 *x, const EVP_PKEY *k)
         break;
     case -2:
         ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE);
+        break;
     }
-    if (ret > 0)
-        return 1;
-    return 0;
+
+    return ret > 0;
 }
 
 /*