PROV & ASYM_CIPHER: Adapt the RSA asymmetric cipher code for PSS-parameters

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index 0ca52ae7439d03881aacb28788b55a40f807b50b..f7e7b549f83126e7d23816f20fa5144d44d2dbf4 100644 (file)
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -95,7 +95,16 @@ static int rsa_init(void *vprsactx, void *vrsa)
         return 0;
     RSA_free(prsactx->rsa);
     prsactx->rsa = vrsa;
-    prsactx->pad_mode = RSA_PKCS1_PADDING;
+
+    switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
+    case RSA_FLAG_TYPE_RSA:
+        prsactx->pad_mode = RSA_PKCS1_PADDING;
+        break;
+    default:
+        ERR_raise(ERR_LIB_PROV, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+        return 0;
+    }
+
     return 1;
 }