EVP: Fix the returned value for ASN1_PKEY_CTRL_DEFAULT_MD_NID

Trust the returned value from EVP_PKEY_get_default_digest_name()!  It
mimics exactly the values that EVP_PKEY_get_default_digest_nid() is
supposed to return, and that value should simply be passed unchanged.
Callers depend on it.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12586)

diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 3e3f2118a26ed5764239acff05fc784d18254bc5..2563cd97caa2299da25b4eebdd5eb9a4fde9ea42 100644 (file)
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -1202,19 +1202,18 @@ static int legacy_asn1_ctrl_to_param(EVP_PKEY *pkey, int op,
     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
         {
             char mdname[80] = "";
-            int nid;
             int rv = EVP_PKEY_get_default_digest_name(pkey, mdname,
                                                       sizeof(mdname));
 
-            if (rv <= 0)
-                return rv;
-            nid = OBJ_sn2nid(mdname);
-            if (nid == NID_undef)
-                nid = OBJ_ln2nid(mdname);
-            if (nid == NID_undef)
-                return 0;
-            *(int *)arg2 = nid;
-            return 1;
+            if (rv > 0) {
+                int nid;
+
+                nid = OBJ_sn2nid(mdname);
+                if (nid == NID_undef)
+                    nid = OBJ_ln2nid(mdname);
+                *(int *)arg2 = nid;
+            }
+            return rv;
         }
     default:
         return -2;