Fix change in behaviour of EVP_PKEY_CTRL_RSA_KEYGEN_BITS

In 1.1.1 the ctrl EVP_PKEY_CTRL_RSA_KEYGEN_BITS would fail immediately
if the number of bits was too small. In 3.0 it always succeeds, and only
fails later during the key generation stage.

We fix that so that it fails early like it used to in 1.1.1.

Note that in 1.1.1 it fails with a -2 return code. That is not the case
in 3.0 and has not been addressed here (see #14442)

Fixes #14443

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14702)

diff --git a/crypto/rsa/rsa_local.h b/crypto/rsa/rsa_local.h
index 6979adfcd1a5825fda2372cfe693e197fa80b5fd..ea70da05ad780e74bae5a9d77d177cebfa8d6a40 100644 (file)
--- a/crypto/rsa/rsa_local.h
+++ b/crypto/rsa/rsa_local.h
@@ -14,7 +14,6 @@
 #include "crypto/rsa.h"
 
 #define RSA_MAX_PRIME_NUM       5
-#define RSA_MIN_MODULUS_BITS    512
 
 typedef struct rsa_prime_info_st {
     BIGNUM *r;

diff --git a/include/crypto/rsa.h b/include/crypto/rsa.h
index 69fa8a4d8acf28dfb01f63eab1db389f692c818a..73bf03f6158cb3716e91c72461361a742e1f2689 100644 (file)
--- a/include/crypto/rsa.h
+++ b/include/crypto/rsa.h
@@ -16,6 +16,8 @@
 # include <openssl/x509.h>
 # include "crypto/types.h"
 
+#define RSA_MIN_MODULUS_BITS    512
+
 typedef struct rsa_pss_params_30_st {
     int hash_algorithm_nid;
     struct {

diff --git a/providers/implementations/keymgmt/rsa_kmgmt.c b/providers/implementations/keymgmt/rsa_kmgmt.c
index eac3843884979628733755942235c6ff728ee4ef..1bcb6ed603ef39beccdc0d1662b2780d005d59d0 100644 (file)
--- a/providers/implementations/keymgmt/rsa_kmgmt.c
+++ b/providers/implementations/keymgmt/rsa_kmgmt.c
@@ -19,6 +19,7 @@
 #include <openssl/err.h>
 #include <openssl/rsa.h>
 #include <openssl/evp.h>
+#include <openssl/proverr.h>
 #include "prov/implementations.h"
 #include "prov/providercommon.h"
 #include "prov/provider_ctx.h"
@@ -473,9 +474,14 @@ static int rsa_gen_set_params(void *genctx, const OSSL_PARAM params[])
     if (params == NULL)
         return 1;
 
-    if ((p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_BITS)) != NULL
-        && !OSSL_PARAM_get_size_t(p, &gctx->nbits))
-        return 0;
+    if ((p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_BITS)) != NULL) {
+        if (!OSSL_PARAM_get_size_t(p, &gctx->nbits))
+            return 0;
+        if (gctx->nbits < RSA_MIN_MODULUS_BITS) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SIZE_TOO_SMALL);
+            return 0;
+        }
+    }
     if ((p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_PRIMES)) != NULL
         && !OSSL_PARAM_get_size_t(p, &gctx->primes))
         return 0;

diff --git a/test/recipes/30-test_evp_data/evppkey_rsa.txt b/test/recipes/30-test_evp_data/evppkey_rsa.txt
index 4354bd649a9288d68a23b54533801e0a63b0c646..79e6715f4c3f216313892ced14721867f3cc3d0c 100644 (file)
--- a/test/recipes/30-test_evp_data/evppkey_rsa.txt
+++ b/test/recipes/30-test_evp_data/evppkey_rsa.txt
@@ -614,5 +614,5 @@ Title = Test RSA keygen
 KeyGen = rsaEncryption
 Ctrl = rsa_keygen_bits:128
 KeyName = tmprsa
-Result = KEYGEN_GENERATE_ERROR
+Result = PKEY_CTRL_ERROR
 Reason = key size too small