From: Dr. David von Oheimb <dev@ddvo.net>
Date: Thu, 7 Nov 2024 21:01:41 +0000 (+0100)
Subject: APPS/pkeyutl: add missing high-level check for -verifyrecover being usable only with RSA
X-Git-Tag: openssl-3.5.0-alpha1~927
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=abad748da8d27508d4cfb74f86786167bd4ba0c4;p=thirdparty%2Fopenssl.git

APPS/pkeyutl: add missing high-level check for -verifyrecover being usable only with RSA

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/25903)
---

diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 64c5d5871a6..34b3e47c80d 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -43,7 +43,7 @@ static int do_raw_keyop(int pkey_op, EVP_MD_CTX *mctx,
                         int filesize, unsigned char *sig, int siglen,
                         unsigned char **out, size_t *poutlen);
 
-static int only_rawin(const EVP_PKEY *pkey)
+static int is_EdDSA(const EVP_PKEY *pkey)
 {
     if (pkey == NULL)
         return 0;
@@ -51,6 +51,11 @@ static int only_rawin(const EVP_PKEY *pkey)
         || EVP_PKEY_is_a(pkey, "ED448");
 }
 
+static int only_rawin(const EVP_PKEY *pkey)
+{
+    return is_EdDSA(pkey);
+}
+
 typedef enum OPTION_choice {
     OPT_COMMON,
     OPT_ENGINE, OPT_ENGINE_IMPL, OPT_IN, OPT_OUT,
@@ -309,10 +314,15 @@ int pkeyutl_main(int argc, char **argv)
     }
 
     pkey = get_pkey(kdfalg, inkey, keyform, key_type, passinarg, pkey_op, e);
+
+    if (pkey_op == EVP_PKEY_OP_VERIFYRECOVER && !EVP_PKEY_is_a(pkey, "RSA")) {
+        BIO_printf(bio_err, "%s: -verifyrecover can be used only with RSA\n", prog);
+        goto end;
+    }
+
     if (pkey_op == EVP_PKEY_OP_SIGN || pkey_op == EVP_PKEY_OP_VERIFY) {
         if (only_rawin(pkey)) {
-            if ((EVP_PKEY_is_a(pkey, "ED25519") || EVP_PKEY_is_a(pkey, "ED448"))
-                && digestname != NULL) {
+            if (is_EdDSA(pkey) && digestname != NULL) {
                 BIO_printf(bio_err,
                            "%s: -digest (prehash) is not supported with EdDSA\n", prog);
                 EVP_PKEY_free(pkey);
diff --git a/test/recipes/20-test_pkeyutl.t b/test/recipes/20-test_pkeyutl.t
index abdbac75412..8795f87b45e 100644
--- a/test/recipes/20-test_pkeyutl.t
+++ b/test/recipes/20-test_pkeyutl.t
@@ -17,7 +17,7 @@ use File::Compare qw/compare_text compare/;
 
 setup("test_pkeyutl");
 
-plan tests => 24;
+plan tests => 25;
 
 # For the tests below we use the cert itself as the TBS file
 
@@ -54,7 +54,7 @@ SKIP: {
 }
 
 SKIP: {
-    skip "Skipping tests that require ECX", 6
+    skip "Skipping tests that require ECX", 7
         if disabled("ecx");
 
     # Ed25519
@@ -68,6 +68,9 @@ SKIP: {
                   '-inkey', srctop_file('test', 'certs', 'server-ed25519-cert.pem'),
                   '-sigfile', 'Ed25519.sig']))),
                   "Verify an Ed25519 signature against a piece of data");
+    ok(!run(app(([ 'openssl', 'pkeyutl', '-verifyrecover', '-in', 'Ed25519.sig',
+                   '-inkey', srctop_file('test', 'certs', 'server-ed25519-key.pem')]))),
+       "Cannot use -verifyrecover with EdDSA");
 
     # Ed448
     ok(run(app(([ 'openssl', 'pkeyutl', '-sign', '-in',