Fix error reporting in EVP_PKEY_{sign,verify,verify_recover}

For some reason, those functions (and the _init functions too) would
raise EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE when the passed
ctx is NULL, and then not check if the provider supplied the function
that would support these libcrypto functions.

This corrects the situation, and has all those libcrypto functions
raise ERR_R_PASS_NULL_PARAMETER if ctx is NULL, and then check for the
corresponding provider supplied, and only when that one is missing,
raise EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE.

Because 0 doesn't mean error for EVP_PKEY_verify(), -1 is returned when
ERR_R_PASSED_NULL_PARAMETER is raised.  This is done consistently for all
affected functions.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23411)

diff --git a/crypto/evp/signature.c b/crypto/evp/signature.c
index 379b344f0da68f2f3de37aa7f4a0c9c393ceb97d..e274c88f6b596eec093bd6a98a79c1e5c764daa5 100644 (file)
--- a/crypto/evp/signature.c
+++ b/crypto/evp/signature.c
@@ -399,8 +399,8 @@ static int evp_pkey_signature_init(EVP_PKEY_CTX *ctx, int operation,
     int iter;
 
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+        ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
+        return -1;
     }
 
     evp_pkey_ctx_free_old_ops(ctx);
@@ -630,8 +630,8 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx,
     int ret;
 
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+        ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
+        return -1;
     }
 
     if (ctx->operation != EVP_PKEY_OP_SIGN) {
@@ -642,6 +642,11 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx,
     if (ctx->op.sig.algctx == NULL)
         goto legacy;
 
+    if (ctx->op.sig.signature->sign == NULL) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+        return -2;
+    }
+
     ret = ctx->op.sig.signature->sign(ctx->op.sig.algctx, sig, siglen,
                                       (sig == NULL) ? 0 : *siglen, tbs, tbslen);
 
@@ -674,8 +679,8 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx,
     int ret;
 
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+        ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
+        return -1;
     }
 
     if (ctx->operation != EVP_PKEY_OP_VERIFY) {
@@ -686,6 +691,11 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx,
     if (ctx->op.sig.algctx == NULL)
         goto legacy;
 
+    if (ctx->op.sig.signature->verify == NULL) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+        return -2;
+    }
+
     ret = ctx->op.sig.signature->verify(ctx->op.sig.algctx, sig, siglen,
                                         tbs, tbslen);
 
@@ -717,8 +727,8 @@ int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx,
     int ret;
 
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+        ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
+        return -1;
     }
 
     if (ctx->operation != EVP_PKEY_OP_VERIFYRECOVER) {
@@ -729,6 +739,11 @@ int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx,
     if (ctx->op.sig.algctx == NULL)
         goto legacy;
 
+    if (ctx->op.sig.signature->verify_recover == NULL) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+        return -2;
+    }
+
     ret = ctx->op.sig.signature->verify_recover(ctx->op.sig.algctx, rout,
                                                 routlen,
                                                 (rout == NULL ? 0 : *routlen),