providers/implementations/signature/rsa.c

   1 /*
   2  * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 /*
  11  * RSA low level APIs are deprecated for public use, but still ok for
  12  * internal use.
  13  */
  14 #include "internal/deprecated.h"
  15
  16 #include "e_os.h" /* strcasecmp */
  17 #include <string.h>
  18 #include <openssl/crypto.h>
  19 #include <openssl/core_dispatch.h>
  20 #include <openssl/core_names.h>
  21 #include <openssl/err.h>
  22 #include <openssl/rsa.h>
  23 #include <openssl/params.h>
  24 #include <openssl/evp.h>
  25 #include "internal/cryptlib.h"
  26 #include "internal/nelem.h"
  27 #include "internal/sizes.h"
  28 #include "crypto/rsa.h"
  29 #include "prov/providercommon.h"
  30 #include "prov/providercommonerr.h"
  31 #include "prov/implementations.h"
  32 #include "prov/provider_ctx.h"
  33 #include "prov/der_rsa.h"
  34 #include "prov/securitycheck.h"
  35
  36 #define RSA_DEFAULT_DIGEST_NAME OSSL_DIGEST_NAME_SHA1
  37
  38 static OSSL_FUNC_signature_newctx_fn rsa_newctx;
  39 static OSSL_FUNC_signature_sign_init_fn rsa_sign_init;
  40 static OSSL_FUNC_signature_verify_init_fn rsa_verify_init;
  41 static OSSL_FUNC_signature_verify_recover_init_fn rsa_verify_recover_init;
  42 static OSSL_FUNC_signature_sign_fn rsa_sign;
  43 static OSSL_FUNC_signature_verify_fn rsa_verify;
  44 static OSSL_FUNC_signature_verify_recover_fn rsa_verify_recover;
  45 static OSSL_FUNC_signature_digest_sign_init_fn rsa_digest_sign_init;
  46 static OSSL_FUNC_signature_digest_sign_update_fn rsa_digest_signverify_update;
  47 static OSSL_FUNC_signature_digest_sign_final_fn rsa_digest_sign_final;
  48 static OSSL_FUNC_signature_digest_verify_init_fn rsa_digest_verify_init;
  49 static OSSL_FUNC_signature_digest_verify_update_fn rsa_digest_signverify_update;
  50 static OSSL_FUNC_signature_digest_verify_final_fn rsa_digest_verify_final;
  51 static OSSL_FUNC_signature_freectx_fn rsa_freectx;
  52 static OSSL_FUNC_signature_dupctx_fn rsa_dupctx;
  53 static OSSL_FUNC_signature_get_ctx_params_fn rsa_get_ctx_params;
  54 static OSSL_FUNC_signature_gettable_ctx_params_fn rsa_gettable_ctx_params;
  55 static OSSL_FUNC_signature_set_ctx_params_fn rsa_set_ctx_params;
  56 static OSSL_FUNC_signature_settable_ctx_params_fn rsa_settable_ctx_params;
  57 static OSSL_FUNC_signature_get_ctx_md_params_fn rsa_get_ctx_md_params;
  58 static OSSL_FUNC_signature_gettable_ctx_md_params_fn rsa_gettable_ctx_md_params;
  59 static OSSL_FUNC_signature_set_ctx_md_params_fn rsa_set_ctx_md_params;
  60 static OSSL_FUNC_signature_settable_ctx_md_params_fn rsa_settable_ctx_md_params;
  61
  62 static OSSL_ITEM padding_item[] = {
  63     { RSA_PKCS1_PADDING,        OSSL_PKEY_RSA_PAD_MODE_PKCSV15 },
  64     { RSA_SSLV23_PADDING,       OSSL_PKEY_RSA_PAD_MODE_SSLV23 },
  65     { RSA_NO_PADDING,           OSSL_PKEY_RSA_PAD_MODE_NONE },
  66     { RSA_X931_PADDING,         OSSL_PKEY_RSA_PAD_MODE_X931 },
  67     { RSA_PKCS1_PSS_PADDING,    OSSL_PKEY_RSA_PAD_MODE_PSS },
  68     { 0,                        NULL     }
  69 };
  70
  71 /*
  72  * What's passed as an actual key is defined by the KEYMGMT interface.
  73  * We happen to know that our KEYMGMT simply passes RSA structures, so
  74  * we use that here too.
  75  */
  76
  77 typedef struct {
  78     OSSL_LIB_CTX *libctx;
  79     char *propq;
  80     RSA *rsa;
  81     int operation;
  82
  83     /*
  84      * Flag to determine if the hash function can be changed (1) or not (0)
  85      * Because it's dangerous to change during a DigestSign or DigestVerify
  86      * operation, this flag is cleared by their Init function, and set again
  87      * by their Final function.
  88      */
  89     unsigned int flag_allow_md : 1;
  90     unsigned int mgf1_md_set : 1;
  91
  92     /* main digest */
  93     EVP_MD *md;
  94     EVP_MD_CTX *mdctx;
  95     int mdnid;
  96     char mdname[OSSL_MAX_NAME_SIZE]; /* Purely informational */
  97
  98     /* RSA padding mode */
  99     int pad_mode;
 100     /* message digest for MGF1 */
 101     EVP_MD *mgf1_md;
 102     int mgf1_mdnid;
 103     char mgf1_mdname[OSSL_MAX_NAME_SIZE]; /* Purely informational */
 104     /* PSS salt length */
 105     int saltlen;
 106     /* Minimum salt length or -1 if no PSS parameter restriction */
 107     int min_saltlen;
 108
 109     /* Temp buffer */
 110     unsigned char *tbuf;
 111
 112 } PROV_RSA_CTX;
 113
 114 /* True if PSS parameters are restricted */
 115 #define rsa_pss_restricted(prsactx) (prsactx->min_saltlen != -1)
 116
 117 static size_t rsa_get_md_size(const PROV_RSA_CTX *prsactx)
 118 {
 119     if (prsactx->md != NULL)
 120         return EVP_MD_size(prsactx->md);
 121     return 0;
 122 }
 123
 124 static int rsa_check_padding(const PROV_RSA_CTX *prsactx,
 125                              const char *mdname, const char *mgf1_mdname,
 126                              int mdnid)
 127 {
 128     switch(prsactx->pad_mode) {
 129         case RSA_NO_PADDING:
 130             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE);
 131             return 0;
 132         case RSA_X931_PADDING:
 133             if (RSA_X931_hash_id(mdnid) == -1) {
 134                 ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_X931_DIGEST);
 135                 return 0;
 136             }
 137             break;
 138         case RSA_PKCS1_PSS_PADDING:
 139             if (rsa_pss_restricted(prsactx))
 140                 if ((mdname != NULL && !EVP_MD_is_a(prsactx->md, mdname))
 141                     || (mgf1_mdname != NULL
 142                         && !EVP_MD_is_a(prsactx->mgf1_md, mgf1_mdname))) {
 143                     ERR_raise(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED);
 144                     return 0;
 145                 }
 146             break;
 147         default:
 148             break;
 149     }
 150
 151     return 1;
 152 }
 153
 154 static int rsa_check_parameters(PROV_RSA_CTX *prsactx, int min_saltlen)
 155 {
 156     if (prsactx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 157         int max_saltlen;
 158
 159         /* See if minimum salt length exceeds maximum possible */
 160         max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(prsactx->md);
 161         if ((RSA_bits(prsactx->rsa) & 0x7) == 1)
 162             max_saltlen--;
 163         if (min_saltlen < 0 || min_saltlen > max_saltlen) {
 164             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
 165             return 0;
 166         }
 167         prsactx->min_saltlen = min_saltlen;
 168     }
 169     return 1;
 170 }
 171
 172 static void *rsa_newctx(void *provctx, const char *propq)
 173 {
 174     PROV_RSA_CTX *prsactx = NULL;
 175     char *propq_copy = NULL;
 176
 177     if (!ossl_prov_is_running())
 178         return NULL;
 179
 180     if ((prsactx = OPENSSL_zalloc(sizeof(PROV_RSA_CTX))) == NULL
 181         || (propq != NULL
 182             && (propq_copy = OPENSSL_strdup(propq)) == NULL)) {
 183         OPENSSL_free(prsactx);
 184         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 185         return NULL;
 186     }
 187
 188     prsactx->libctx = PROV_LIBCTX_OF(provctx);
 189     prsactx->flag_allow_md = 1;
 190     prsactx->propq = propq_copy;
 191     return prsactx;
 192 }
 193
 194 static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx)
 195 {
 196     int saltlen = ctx->saltlen;
 197
 198     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
 199         saltlen = EVP_MD_size(ctx->md);
 200     } else if (saltlen == RSA_PSS_SALTLEN_AUTO || saltlen == RSA_PSS_SALTLEN_MAX) {
 201         saltlen = RSA_size(ctx->rsa) - EVP_MD_size(ctx->md) - 2;
 202         if ((RSA_bits(ctx->rsa) & 0x7) == 1)
 203             saltlen--;
 204     }
 205     if (saltlen < 0) {
 206         ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
 207         return -1;
 208     } else if (saltlen < ctx->min_saltlen) {
 209         ERR_raise_data(ERR_LIB_PROV, PROV_R_PSS_SALTLEN_TOO_SMALL,
 210                        "minimum salt length: %d, actual salt length: %d",
 211                        ctx->min_saltlen, saltlen);
 212         return -1;
 213     }
 214     return saltlen;
 215 }
 216
 217 static unsigned char *rsa_generate_signature_aid(PROV_RSA_CTX *ctx,
 218                                                  unsigned char *aid_buf,
 219                                                  size_t buf_len,
 220                                                  size_t *aid_len)
 221 {
 222     WPACKET pkt;
 223     unsigned char *aid = NULL;
 224     int saltlen;
 225     RSA_PSS_PARAMS_30 pss_params;
 226
 227     if (!WPACKET_init_der(&pkt, aid_buf, buf_len)) {
 228         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 229         return NULL;
 230     }
 231
 232     switch(ctx->pad_mode) {
 233         case RSA_PKCS1_PADDING:
 234             if (!ossl_DER_w_algorithmIdentifier_MDWithRSAEncryption(&pkt, -1,
 235                                                                     ctx->mdnid)) {
 236                 ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
 237                 goto cleanup;
 238             }
 239             break;
 240         case RSA_PKCS1_PSS_PADDING:
 241             saltlen = rsa_pss_compute_saltlen(ctx);
 242             if (saltlen < 0)
 243                 goto cleanup;
 244             if (!ossl_rsa_pss_params_30_set_defaults(&pss_params)
 245                 || !ossl_rsa_pss_params_30_set_hashalg(&pss_params, ctx->mdnid)
 246                 || !ossl_rsa_pss_params_30_set_maskgenhashalg(&pss_params,
 247                                                               ctx->mgf1_mdnid)
 248                 || !ossl_rsa_pss_params_30_set_saltlen(&pss_params, saltlen)
 249                 || !ossl_DER_w_algorithmIdentifier_RSA_PSS(&pkt, -1,
 250                                                            RSA_FLAG_TYPE_RSASSAPSS,
 251                                                            &pss_params)) {
 252                 ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
 253                 goto cleanup;
 254             }
 255             break;
 256         default:
 257             ERR_raise_data(ERR_LIB_PROV, ERR_R_UNSUPPORTED,
 258                            "Algorithm ID generation");
 259             goto cleanup;
 260     }
 261     if (WPACKET_finish(&pkt)) {
 262         WPACKET_get_total_written(&pkt, aid_len);
 263         aid = WPACKET_get_curr(&pkt);
 264     }
 265  cleanup:
 266     WPACKET_cleanup(&pkt);
 267     return aid;
 268 }
 269
 270 static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
 271                         const char *mdprops)
 272 {
 273     if (mdprops == NULL)
 274         mdprops = ctx->propq;
 275
 276     if (mdname != NULL) {
 277         EVP_MD *md = EVP_MD_fetch(ctx->libctx, mdname, mdprops);
 278         int sha1_allowed = (ctx->operation != EVP_PKEY_OP_SIGN);
 279         int md_nid = digest_rsa_sign_get_md_nid(md, sha1_allowed);
 280         size_t mdname_len = strlen(mdname);
 281
 282         if (md == NULL
 283             || md_nid == NID_undef
 284             || !rsa_check_padding(ctx, mdname, NULL, md_nid)
 285             || mdname_len >= sizeof(ctx->mdname)) {
 286             if (md == NULL)
 287                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 288                                "%s could not be fetched", mdname);
 289             if (md_nid == NID_undef)
 290                 ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
 291                                "digest=%s", mdname);
 292             if (mdname_len >= sizeof(ctx->mdname))
 293                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 294                                "%s exceeds name buffer length", mdname);
 295             EVP_MD_free(md);
 296             return 0;
 297         }
 298
 299         if (!ctx->mgf1_md_set) {
 300             if (!EVP_MD_up_ref(md)) {
 301                 EVP_MD_free(md);
 302                 return 0;
 303             }
 304             EVP_MD_free(ctx->mgf1_md);
 305             ctx->mgf1_md = md;
 306             ctx->mgf1_mdnid = md_nid;
 307             OPENSSL_strlcpy(ctx->mgf1_mdname, mdname, sizeof(ctx->mgf1_mdname));
 308         }
 309
 310         EVP_MD_CTX_free(ctx->mdctx);
 311         EVP_MD_free(ctx->md);
 312
 313         ctx->mdctx = NULL;
 314         ctx->md = md;
 315         ctx->mdnid = md_nid;
 316         OPENSSL_strlcpy(ctx->mdname, mdname, sizeof(ctx->mdname));
 317     }
 318
 319     return 1;
 320 }
 321
 322 static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
 323                              const char *mdprops)
 324 {
 325     size_t len;
 326     EVP_MD *md = NULL;
 327     int mdnid;
 328
 329     if (mdprops == NULL)
 330         mdprops = ctx->propq;
 331
 332     if ((md = EVP_MD_fetch(ctx->libctx, mdname, mdprops)) == NULL) {
 333         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 334                        "%s could not be fetched", mdname);
 335         return 0;
 336     }
 337     /* The default for mgf1 is SHA1 - so allow SHA1 */
 338     if ((mdnid = digest_rsa_sign_get_md_nid(md, 1)) == NID_undef
 339         || !rsa_check_padding(ctx, NULL, mdname, mdnid)) {
 340         if (mdnid == NID_undef)
 341             ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
 342                            "digest=%s", mdname);
 343         EVP_MD_free(md);
 344         return 0;
 345     }
 346     len = OPENSSL_strlcpy(ctx->mgf1_mdname, mdname, sizeof(ctx->mgf1_mdname));
 347     if (len >= sizeof(ctx->mgf1_mdname)) {
 348         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 349                        "%s exceeds name buffer length", mdname);
 350         EVP_MD_free(md);
 351         return 0;
 352     }
 353
 354     EVP_MD_free(ctx->mgf1_md);
 355     ctx->mgf1_md = md;
 356     ctx->mgf1_mdnid = mdnid;
 357     ctx->mgf1_md_set = 1;
 358     return 1;
 359 }
 360
 361 static int rsa_signverify_init(void *vprsactx, void *vrsa, int operation)
 362 {
 363     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 364
 365     if (!ossl_prov_is_running())
 366         return 0;
 367
 368     if (prsactx == NULL || vrsa == NULL || !RSA_up_ref(vrsa))
 369         return 0;
 370
 371     RSA_free(prsactx->rsa);
 372     prsactx->rsa = vrsa;
 373     prsactx->operation = operation;
 374
 375     if (!ossl_rsa_check_key(vrsa, operation == EVP_PKEY_OP_SIGN)) {
 376         ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH);
 377         return 0;
 378     }
 379
 380     /* Maximum for sign, auto for verify */
 381     prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
 382     prsactx->min_saltlen = -1;
 383
 384     switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
 385     case RSA_FLAG_TYPE_RSA:
 386         prsactx->pad_mode = RSA_PKCS1_PADDING;
 387         break;
 388     case RSA_FLAG_TYPE_RSASSAPSS:
 389         prsactx->pad_mode = RSA_PKCS1_PSS_PADDING;
 390
 391         {
 392             const RSA_PSS_PARAMS_30 *pss =
 393                 ossl_rsa_get0_pss_params_30(prsactx->rsa);
 394
 395             if (!ossl_rsa_pss_params_30_is_unrestricted(pss)) {
 396                 int md_nid = ossl_rsa_pss_params_30_hashalg(pss);
 397                 int mgf1md_nid = ossl_rsa_pss_params_30_maskgenhashalg(pss);
 398                 int min_saltlen = ossl_rsa_pss_params_30_saltlen(pss);
 399                 const char *mdname, *mgf1mdname;
 400                 size_t len;
 401
 402                 mdname = ossl_rsa_oaeppss_nid2name(md_nid);
 403                 mgf1mdname = ossl_rsa_oaeppss_nid2name(mgf1md_nid);
 404
 405                 if (mdname == NULL) {
 406                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 407                                    "PSS restrictions lack hash algorithm");
 408                     return 0;
 409                 }
 410                 if (mgf1mdname == NULL) {
 411                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 412                                    "PSS restrictions lack MGF1 hash algorithm");
 413                     return 0;
 414                 }
 415
 416                 len = OPENSSL_strlcpy(prsactx->mdname, mdname,
 417                                       sizeof(prsactx->mdname));
 418                 if (len >= sizeof(prsactx->mdname)) {
 419                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 420                                    "hash algorithm name too long");
 421                     return 0;
 422                 }
 423                 len = OPENSSL_strlcpy(prsactx->mgf1_mdname, mgf1mdname,
 424                                       sizeof(prsactx->mgf1_mdname));
 425                 if (len >= sizeof(prsactx->mgf1_mdname)) {
 426                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 427                                    "MGF1 hash algorithm name too long");
 428                     return 0;
 429                 }
 430                 prsactx->saltlen = min_saltlen;
 431
 432                 /* call rsa_setup_mgf1_md before rsa_setup_md to avoid duplication */
 433                 return rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq)
 434                     && rsa_setup_md(prsactx, mdname, prsactx->propq)
 435                     && rsa_check_parameters(prsactx, min_saltlen);
 436             }
 437         }
 438
 439         break;
 440     default:
 441         ERR_raise(ERR_LIB_RSA, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
 442         return 0;
 443     }
 444
 445     return 1;
 446 }
 447
 448 static int setup_tbuf(PROV_RSA_CTX *ctx)
 449 {
 450     if (ctx->tbuf != NULL)
 451         return 1;
 452     if ((ctx->tbuf = OPENSSL_malloc(RSA_size(ctx->rsa))) == NULL) {
 453         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 454         return 0;
 455     }
 456     return 1;
 457 }
 458
 459 static void clean_tbuf(PROV_RSA_CTX *ctx)
 460 {
 461     if (ctx->tbuf != NULL)
 462         OPENSSL_cleanse(ctx->tbuf, RSA_size(ctx->rsa));
 463 }
 464
 465 static void free_tbuf(PROV_RSA_CTX *ctx)
 466 {
 467     clean_tbuf(ctx);
 468     OPENSSL_free(ctx->tbuf);
 469     ctx->tbuf = NULL;
 470 }
 471
 472 static int rsa_sign_init(void *vprsactx, void *vrsa)
 473 {
 474     if (!ossl_prov_is_running())
 475         return 0;
 476     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_SIGN);
 477 }
 478
 479 static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen,
 480                     size_t sigsize, const unsigned char *tbs, size_t tbslen)
 481 {
 482     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 483     int ret;
 484     size_t rsasize = RSA_size(prsactx->rsa);
 485     size_t mdsize = rsa_get_md_size(prsactx);
 486
 487     if (!ossl_prov_is_running())
 488         return 0;
 489
 490     if (sig == NULL) {
 491         *siglen = rsasize;
 492         return 1;
 493     }
 494
 495     if (sigsize < rsasize) {
 496         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SIGNATURE_SIZE,
 497                        "is %zu, should be at least %zu", sigsize, rsasize);
 498         return 0;
 499     }
 500
 501     if (mdsize != 0) {
 502         if (tbslen != mdsize) {
 503             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH);
 504             return 0;
 505         }
 506
 507 #ifndef FIPS_MODULE
 508         if (EVP_MD_is_a(prsactx->md, OSSL_DIGEST_NAME_MDC2)) {
 509             unsigned int sltmp;
 510
 511             if (prsactx->pad_mode != RSA_PKCS1_PADDING) {
 512                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 513                                "only PKCS#1 padding supported with MDC2");
 514                 return 0;
 515             }
 516             ret = RSA_sign_ASN1_OCTET_STRING(0, tbs, tbslen, sig, &sltmp,
 517                                              prsactx->rsa);
 518
 519             if (ret <= 0) {
 520                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 521                 return 0;
 522             }
 523             ret = sltmp;
 524             goto end;
 525         }
 526 #endif
 527         switch (prsactx->pad_mode) {
 528         case RSA_X931_PADDING:
 529             if ((size_t)RSA_size(prsactx->rsa) < tbslen + 1) {
 530                 ERR_raise_data(ERR_LIB_PROV, PROV_R_KEY_SIZE_TOO_SMALL,
 531                                "RSA key size = %d, expected minimum = %d",
 532                                RSA_size(prsactx->rsa), tbslen + 1);
 533                 return 0;
 534             }
 535             if (!setup_tbuf(prsactx)) {
 536                 ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 537                 return 0;
 538             }
 539             memcpy(prsactx->tbuf, tbs, tbslen);
 540             prsactx->tbuf[tbslen] = RSA_X931_hash_id(prsactx->mdnid);
 541             ret = RSA_private_encrypt(tbslen + 1, prsactx->tbuf,
 542                                       sig, prsactx->rsa, RSA_X931_PADDING);
 543             clean_tbuf(prsactx);
 544             break;
 545
 546         case RSA_PKCS1_PADDING:
 547             {
 548                 unsigned int sltmp;
 549
 550                 ret = RSA_sign(prsactx->mdnid, tbs, tbslen, sig, &sltmp,
 551                                prsactx->rsa);
 552                 if (ret <= 0) {
 553                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 554                     return 0;
 555                 }
 556                 ret = sltmp;
 557             }
 558             break;
 559
 560         case RSA_PKCS1_PSS_PADDING:
 561             /* Check PSS restrictions */
 562             if (rsa_pss_restricted(prsactx)) {
 563                 switch (prsactx->saltlen) {
 564                 case RSA_PSS_SALTLEN_DIGEST:
 565                     if (prsactx->min_saltlen > EVP_MD_size(prsactx->md)) {
 566                         ERR_raise_data(ERR_LIB_PROV,
 567                                        PROV_R_PSS_SALTLEN_TOO_SMALL,
 568                                        "minimum salt length set to %d, "
 569                                        "but the digest only gives %d",
 570                                        prsactx->min_saltlen,
 571                                        EVP_MD_size(prsactx->md));
 572                         return 0;
 573                     }
 574                     /* FALLTHRU */
 575                 default:
 576                     if (prsactx->saltlen >= 0
 577                         && prsactx->saltlen < prsactx->min_saltlen) {
 578                         ERR_raise_data(ERR_LIB_PROV,
 579                                        PROV_R_PSS_SALTLEN_TOO_SMALL,
 580                                        "minimum salt length set to %d, but the"
 581                                        "actual salt length is only set to %d",
 582                                        prsactx->min_saltlen,
 583                                        prsactx->saltlen);
 584                         return 0;
 585                     }
 586                     break;
 587                 }
 588             }
 589             if (!setup_tbuf(prsactx))
 590                 return 0;
 591             if (!RSA_padding_add_PKCS1_PSS_mgf1(prsactx->rsa,
 592                                                 prsactx->tbuf, tbs,
 593                                                 prsactx->md, prsactx->mgf1_md,
 594                                                 prsactx->saltlen)) {
 595                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 596                 return 0;
 597             }
 598             ret = RSA_private_encrypt(RSA_size(prsactx->rsa), prsactx->tbuf,
 599                                       sig, prsactx->rsa, RSA_NO_PADDING);
 600             clean_tbuf(prsactx);
 601             break;
 602
 603         default:
 604             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 605                            "Only X.931, PKCS#1 v1.5 or PSS padding allowed");
 606             return 0;
 607         }
 608     } else {
 609         ret = RSA_private_encrypt(tbslen, tbs, sig, prsactx->rsa,
 610                                   prsactx->pad_mode);
 611     }
 612
 613 #ifndef FIPS_MODULE
 614  end:
 615 #endif
 616     if (ret <= 0) {
 617         ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 618         return 0;
 619     }
 620
 621     *siglen = ret;
 622     return 1;
 623 }
 624
 625 static int rsa_verify_recover_init(void *vprsactx, void *vrsa)
 626 {
 627     if (!ossl_prov_is_running())
 628         return 0;
 629     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFYRECOVER);
 630 }
 631
 632 static int rsa_verify_recover(void *vprsactx,
 633                               unsigned char *rout,
 634                               size_t *routlen,
 635                               size_t routsize,
 636                               const unsigned char *sig,
 637                               size_t siglen)
 638 {
 639     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 640     int ret;
 641
 642     if (!ossl_prov_is_running())
 643         return 0;
 644
 645     if (rout == NULL) {
 646         *routlen = RSA_size(prsactx->rsa);
 647         return 1;
 648     }
 649
 650     if (prsactx->md != NULL) {
 651         switch (prsactx->pad_mode) {
 652         case RSA_X931_PADDING:
 653             if (!setup_tbuf(prsactx))
 654                 return 0;
 655             ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
 656                                      RSA_X931_PADDING);
 657             if (ret < 1) {
 658                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 659                 return 0;
 660             }
 661             ret--;
 662             if (prsactx->tbuf[ret] != RSA_X931_hash_id(prsactx->mdnid)) {
 663                 ERR_raise(ERR_LIB_PROV, PROV_R_ALGORITHM_MISMATCH);
 664                 return 0;
 665             }
 666             if (ret != EVP_MD_size(prsactx->md)) {
 667                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH,
 668                                "Should be %d, but got %d",
 669                                EVP_MD_size(prsactx->md), ret);
 670                 return 0;
 671             }
 672
 673             *routlen = ret;
 674             if (rout != prsactx->tbuf) {
 675                 if (routsize < (size_t)ret) {
 676                     ERR_raise_data(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL,
 677                                    "buffer size is %d, should be %d",
 678                                    routsize, ret);
 679                     return 0;
 680                 }
 681                 memcpy(rout, prsactx->tbuf, ret);
 682             }
 683             break;
 684
 685         case RSA_PKCS1_PADDING:
 686             {
 687                 size_t sltmp;
 688
 689                 ret = int_rsa_verify(prsactx->mdnid, NULL, 0, rout, &sltmp,
 690                                      sig, siglen, prsactx->rsa);
 691                 if (ret <= 0) {
 692                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 693                     return 0;
 694                 }
 695                 ret = sltmp;
 696             }
 697             break;
 698
 699         default:
 700             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 701                            "Only X.931 or PKCS#1 v1.5 padding allowed");
 702             return 0;
 703         }
 704     } else {
 705         ret = RSA_public_decrypt(siglen, sig, rout, prsactx->rsa,
 706                                  prsactx->pad_mode);
 707         if (ret < 0) {
 708             ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 709             return 0;
 710         }
 711     }
 712     *routlen = ret;
 713     return 1;
 714 }
 715
 716 static int rsa_verify_init(void *vprsactx, void *vrsa)
 717 {
 718     if (!ossl_prov_is_running())
 719         return 0;
 720     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFY);
 721 }
 722
 723 static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
 724                       const unsigned char *tbs, size_t tbslen)
 725 {
 726     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 727     size_t rslen;
 728
 729     if (!ossl_prov_is_running())
 730         return 0;
 731     if (prsactx->md != NULL) {
 732         switch (prsactx->pad_mode) {
 733         case RSA_PKCS1_PADDING:
 734             if (!RSA_verify(prsactx->mdnid, tbs, tbslen, sig, siglen,
 735                             prsactx->rsa)) {
 736                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 737                 return 0;
 738             }
 739             return 1;
 740         case RSA_X931_PADDING:
 741             if (!setup_tbuf(prsactx))
 742                 return 0;
 743             if (rsa_verify_recover(prsactx, prsactx->tbuf, &rslen, 0,
 744                                    sig, siglen) <= 0)
 745                 return 0;
 746             break;
 747         case RSA_PKCS1_PSS_PADDING:
 748             {
 749                 int ret;
 750                 size_t mdsize;
 751
 752                 /*
 753                  * We need to check this for the RSA_verify_PKCS1_PSS_mgf1()
 754                  * call
 755                  */
 756                 mdsize = rsa_get_md_size(prsactx);
 757                 if (tbslen != mdsize) {
 758                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH,
 759                                    "Should be %d, but got %d",
 760                                    mdsize, tbslen);
 761                     return 0;
 762                 }
 763
 764                 if (!setup_tbuf(prsactx))
 765                     return 0;
 766                 ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf,
 767                                          prsactx->rsa, RSA_NO_PADDING);
 768                 if (ret <= 0) {
 769                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 770                     return 0;
 771                 }
 772                 ret = RSA_verify_PKCS1_PSS_mgf1(prsactx->rsa, tbs,
 773                                                 prsactx->md, prsactx->mgf1_md,
 774                                                 prsactx->tbuf,
 775                                                 prsactx->saltlen);
 776                 if (ret <= 0) {
 777                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 778                     return 0;
 779                 }
 780                 return 1;
 781             }
 782         default:
 783             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 784                            "Only X.931, PKCS#1 v1.5 or PSS padding allowed");
 785             return 0;
 786         }
 787     } else {
 788         if (!setup_tbuf(prsactx))
 789             return 0;
 790         rslen = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
 791                                    prsactx->pad_mode);
 792         if (rslen == 0) {
 793             ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 794             return 0;
 795         }
 796     }
 797
 798     if ((rslen != tbslen) || memcmp(tbs, prsactx->tbuf, rslen))
 799         return 0;
 800
 801     return 1;
 802 }
 803
 804 static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
 805                                       void *vrsa, int operation)
 806 {
 807     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 808
 809     if (!ossl_prov_is_running())
 810         return 0;
 811
 812     if (prsactx != NULL)
 813         prsactx->flag_allow_md = 0;
 814     if (!rsa_signverify_init(vprsactx, vrsa, operation))
 815         return 0;
 816     if (mdname != NULL
 817         /* was rsa_setup_md already called in rsa_signverify_init()? */
 818         && (mdname[0] == '\0' || strcasecmp(prsactx->mdname, mdname) != 0)
 819         && !rsa_setup_md(prsactx, mdname, prsactx->propq))
 820         return 0;
 821
 822     prsactx->mdctx = EVP_MD_CTX_new();
 823     if (prsactx->mdctx == NULL) {
 824         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 825         goto error;
 826     }
 827
 828     if (!EVP_DigestInit_ex(prsactx->mdctx, prsactx->md, NULL))
 829         goto error;
 830
 831     return 1;
 832
 833  error:
 834     EVP_MD_CTX_free(prsactx->mdctx);
 835     EVP_MD_free(prsactx->md);
 836     prsactx->mdctx = NULL;
 837     prsactx->md = NULL;
 838     return 0;
 839 }
 840
 841 static int rsa_digest_signverify_update(void *vprsactx,
 842                                         const unsigned char *data,
 843                                         size_t datalen)
 844 {
 845     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 846
 847     if (prsactx == NULL || prsactx->mdctx == NULL)
 848         return 0;
 849
 850     return EVP_DigestUpdate(prsactx->mdctx, data, datalen);
 851 }
 852
 853 static int rsa_digest_sign_init(void *vprsactx, const char *mdname,
 854                                 void *vrsa)
 855 {
 856     if (!ossl_prov_is_running())
 857         return 0;
 858     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
 859                                       EVP_PKEY_OP_SIGN);
 860 }
 861
 862 static int rsa_digest_sign_final(void *vprsactx, unsigned char *sig,
 863                                  size_t *siglen, size_t sigsize)
 864 {
 865     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 866     unsigned char digest[EVP_MAX_MD_SIZE];
 867     unsigned int dlen = 0;
 868
 869     if (!ossl_prov_is_running() || prsactx == NULL)
 870         return 0;
 871     prsactx->flag_allow_md = 1;
 872     if (prsactx->mdctx == NULL)
 873         return 0;
 874     /*
 875      * If sig is NULL then we're just finding out the sig size. Other fields
 876      * are ignored. Defer to rsa_sign.
 877      */
 878     if (sig != NULL) {
 879         /*
 880          * The digests used here are all known (see rsa_get_md_nid()), so they
 881          * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
 882          */
 883         if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
 884             return 0;
 885     }
 886
 887     return rsa_sign(vprsactx, sig, siglen, sigsize, digest, (size_t)dlen);
 888 }
 889
 890 static int rsa_digest_verify_init(void *vprsactx, const char *mdname,
 891                                   void *vrsa)
 892 {
 893     if (!ossl_prov_is_running())
 894         return 0;
 895     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
 896                                       EVP_PKEY_OP_VERIFY);
 897 }
 898
 899 int rsa_digest_verify_final(void *vprsactx, const unsigned char *sig,
 900                             size_t siglen)
 901 {
 902     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 903     unsigned char digest[EVP_MAX_MD_SIZE];
 904     unsigned int dlen = 0;
 905
 906     if (!ossl_prov_is_running())
 907         return 0;
 908
 909     if (prsactx == NULL)
 910         return 0;
 911     prsactx->flag_allow_md = 1;
 912     if (prsactx->mdctx == NULL)
 913         return 0;
 914
 915     /*
 916      * The digests used here are all known (see rsa_get_md_nid()), so they
 917      * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
 918      */
 919     if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
 920         return 0;
 921
 922     return rsa_verify(vprsactx, sig, siglen, digest, (size_t)dlen);
 923 }
 924
 925 static void rsa_freectx(void *vprsactx)
 926 {
 927     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 928
 929     if (prsactx == NULL)
 930         return;
 931
 932     EVP_MD_CTX_free(prsactx->mdctx);
 933     EVP_MD_free(prsactx->md);
 934     EVP_MD_free(prsactx->mgf1_md);
 935     OPENSSL_free(prsactx->propq);
 936     free_tbuf(prsactx);
 937     RSA_free(prsactx->rsa);
 938
 939     OPENSSL_clear_free(prsactx, sizeof(*prsactx));
 940 }
 941
 942 static void *rsa_dupctx(void *vprsactx)
 943 {
 944     PROV_RSA_CTX *srcctx = (PROV_RSA_CTX *)vprsactx;
 945     PROV_RSA_CTX *dstctx;
 946
 947     if (!ossl_prov_is_running())
 948         return NULL;
 949
 950     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
 951     if (dstctx == NULL) {
 952         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 953         return NULL;
 954     }
 955
 956     *dstctx = *srcctx;
 957     dstctx->rsa = NULL;
 958     dstctx->md = NULL;
 959     dstctx->mdctx = NULL;
 960     dstctx->tbuf = NULL;
 961     dstctx->propq = NULL;
 962
 963     if (srcctx->rsa != NULL && !RSA_up_ref(srcctx->rsa))
 964         goto err;
 965     dstctx->rsa = srcctx->rsa;
 966
 967     if (srcctx->md != NULL && !EVP_MD_up_ref(srcctx->md))
 968         goto err;
 969     dstctx->md = srcctx->md;
 970
 971     if (srcctx->mgf1_md != NULL && !EVP_MD_up_ref(srcctx->mgf1_md))
 972         goto err;
 973     dstctx->mgf1_md = srcctx->mgf1_md;
 974
 975     if (srcctx->mdctx != NULL) {
 976         dstctx->mdctx = EVP_MD_CTX_new();
 977         if (dstctx->mdctx == NULL
 978                 || !EVP_MD_CTX_copy_ex(dstctx->mdctx, srcctx->mdctx))
 979             goto err;
 980     }
 981
 982     if (srcctx->propq != NULL) {
 983         dstctx->propq = OPENSSL_strdup(srcctx->propq);
 984         if (dstctx->propq == NULL)
 985             goto err;
 986     }
 987
 988     return dstctx;
 989  err:
 990     rsa_freectx(dstctx);
 991     return NULL;
 992 }
 993
 994 static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params)
 995 {
 996     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 997     OSSL_PARAM *p;
 998
 999     if (prsactx == NULL || params == NULL)
1000         return 0;
1001
1002     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_ALGORITHM_ID);
1003     if (p != NULL) {
1004         /* The Algorithm Identifier of the combined signature algorithm */
1005         unsigned char aid_buf[128];
1006         unsigned char *aid;
1007         size_t  aid_len;
1008
1009         aid = rsa_generate_signature_aid(prsactx, aid_buf,
1010                                          sizeof(aid_buf), &aid_len);
1011         if (aid == NULL || !OSSL_PARAM_set_octet_string(p, aid, aid_len))
1012             return 0;
1013     }
1014
1015     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_PAD_MODE);
1016     if (p != NULL)
1017         switch (p->data_type) {
1018         case OSSL_PARAM_INTEGER:
1019             if (!OSSL_PARAM_set_int(p, prsactx->pad_mode))
1020                 return 0;
1021             break;
1022         case OSSL_PARAM_UTF8_STRING:
1023             {
1024                 int i;
1025                 const char *word = NULL;
1026
1027                 for (i = 0; padding_item[i].id != 0; i++) {
1028                     if (prsactx->pad_mode == (int)padding_item[i].id) {
1029                         word = padding_item[i].ptr;
1030                         break;
1031                     }
1032                 }
1033
1034                 if (word != NULL) {
1035                     if (!OSSL_PARAM_set_utf8_string(p, word))
1036                         return 0;
1037                 } else {
1038                     ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
1039                 }
1040             }
1041             break;
1042         default:
1043             return 0;
1044         }
1045
1046     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_DIGEST);
1047     if (p != NULL && !OSSL_PARAM_set_utf8_string(p, prsactx->mdname))
1048         return 0;
1049
1050     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_MGF1_DIGEST);
1051     if (p != NULL && !OSSL_PARAM_set_utf8_string(p, prsactx->mgf1_mdname))
1052         return 0;
1053
1054     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_PSS_SALTLEN);
1055     if (p != NULL) {
1056         if (p->data_type == OSSL_PARAM_INTEGER) {
1057             if (!OSSL_PARAM_set_int(p, prsactx->saltlen))
1058                 return 0;
1059         } else if (p->data_type == OSSL_PARAM_UTF8_STRING) {
1060             const char *value = NULL;
1061
1062             switch (prsactx->saltlen) {
1063             case RSA_PSS_SALTLEN_DIGEST:
1064                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST;
1065                 break;
1066             case RSA_PSS_SALTLEN_MAX:
1067                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_MAX;
1068                 break;
1069             case RSA_PSS_SALTLEN_AUTO:
1070                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO;
1071                 break;
1072             default:
1073                 {
1074                     int len = BIO_snprintf(p->data, p->data_size, "%d",
1075                                            prsactx->saltlen);
1076
1077                     if (len <= 0)
1078                         return 0;
1079                     p->return_size = len;
1080                     break;
1081                 }
1082             }
1083             if (value != NULL
1084                 && !OSSL_PARAM_set_utf8_string(p, value))
1085                 return 0;
1086         }
1087     }
1088
1089     return 1;
1090 }
1091
1092 static const OSSL_PARAM known_gettable_ctx_params[] = {
1093     OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_ALGORITHM_ID, NULL, 0),
1094     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PAD_MODE, NULL, 0),
1095     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0),
1096     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_DIGEST, NULL, 0),
1097     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PSS_SALTLEN, NULL, 0),
1098     OSSL_PARAM_END
1099 };
1100
1101 static const OSSL_PARAM *rsa_gettable_ctx_params(ossl_unused void *vctx)
1102 {
1103     return known_gettable_ctx_params;
1104 }
1105
1106 static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
1107 {
1108     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1109     const OSSL_PARAM *p;
1110     int pad_mode = prsactx->pad_mode;
1111     int saltlen = prsactx->saltlen;
1112     char mdname[OSSL_MAX_NAME_SIZE] = "", *pmdname = NULL;
1113     char mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmdprops = NULL;
1114     char mgf1mdname[OSSL_MAX_NAME_SIZE] = "", *pmgf1mdname = NULL;
1115     char mgf1mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmgf1mdprops = NULL;
1116
1117     if (prsactx == NULL || params == NULL)
1118         return 0;
1119
1120     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST);
1121     /* Not allowed during certain operations */
1122     if (p != NULL && !prsactx->flag_allow_md)
1123         return 0;
1124     if (p != NULL) {
1125         const OSSL_PARAM *propsp =
1126             OSSL_PARAM_locate_const(params,
1127                                     OSSL_SIGNATURE_PARAM_PROPERTIES);
1128
1129         pmdname = mdname;
1130         if (!OSSL_PARAM_get_utf8_string(p, &pmdname, sizeof(mdname)))
1131             return 0;
1132
1133         if (propsp != NULL) {
1134             pmdprops = mdprops;
1135             if (!OSSL_PARAM_get_utf8_string(propsp,
1136                                             &pmdprops, sizeof(mdprops)))
1137                 return 0;
1138         }
1139     }
1140
1141     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_PAD_MODE);
1142     if (p != NULL) {
1143         const char *err_extra_text = NULL;
1144
1145         switch (p->data_type) {
1146         case OSSL_PARAM_INTEGER: /* Support for legacy pad mode number */
1147             if (!OSSL_PARAM_get_int(p, &pad_mode))
1148                 return 0;
1149             break;
1150         case OSSL_PARAM_UTF8_STRING:
1151             {
1152                 int i;
1153
1154                 if (p->data == NULL)
1155                     return 0;
1156
1157                 for (i = 0; padding_item[i].id != 0; i++) {
1158                     if (strcmp(p->data, padding_item[i].ptr) == 0) {
1159                         pad_mode = padding_item[i].id;
1160                         break;
1161                     }
1162                 }
1163             }
1164             break;
1165         default:
1166             return 0;
1167         }
1168
1169         switch (pad_mode) {
1170         case RSA_PKCS1_OAEP_PADDING:
1171             /*
1172              * OAEP padding is for asymmetric cipher only so is not compatible
1173              * with signature use.
1174              */
1175             err_extra_text = "OAEP padding not allowed for signing / verifying";
1176             goto bad_pad;
1177         case RSA_PKCS1_PSS_PADDING:
1178             if ((prsactx->operation
1179                  & (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)) == 0) {
1180                 err_extra_text =
1181                     "PSS padding only allowed for sign and verify operations";
1182                 goto bad_pad;
1183             }
1184             break;
1185         case RSA_PKCS1_PADDING:
1186             err_extra_text = "PKCS#1 padding not allowed with RSA-PSS";
1187             goto cont;
1188         case RSA_SSLV23_PADDING:
1189             err_extra_text = "SSLv3 padding not allowed with RSA-PSS";
1190             goto cont;
1191         case RSA_NO_PADDING:
1192             err_extra_text = "No padding not allowed with RSA-PSS";
1193             goto cont;
1194         case RSA_X931_PADDING:
1195             err_extra_text = "X.931 padding not allowed with RSA-PSS";
1196         cont:
1197             if (RSA_test_flags(prsactx->rsa,
1198                                RSA_FLAG_TYPE_MASK) == RSA_FLAG_TYPE_RSA)
1199                 break;
1200             /* FALLTHRU */
1201         default:
1202         bad_pad:
1203             if (err_extra_text == NULL)
1204                 ERR_raise(ERR_LIB_PROV,
1205                           PROV_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
1206             else
1207                 ERR_raise_data(ERR_LIB_PROV,
1208                                PROV_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE,
1209                                err_extra_text);
1210             return 0;
1211         }
1212     }
1213
1214     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_PSS_SALTLEN);
1215     if (p != NULL) {
1216         if (pad_mode != RSA_PKCS1_PSS_PADDING) {
1217             ERR_raise_data(ERR_LIB_PROV, PROV_R_NOT_SUPPORTED,
1218                            "PSS saltlen can only be specified if "
1219                            "PSS padding has been specified first");
1220             return 0;
1221         }
1222
1223         switch (p->data_type) {
1224         case OSSL_PARAM_INTEGER: /* Support for legacy pad mode number */
1225             if (!OSSL_PARAM_get_int(p, &saltlen))
1226                 return 0;
1227             break;
1228         case OSSL_PARAM_UTF8_STRING:
1229             if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST) == 0)
1230                 saltlen = RSA_PSS_SALTLEN_DIGEST;
1231             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_MAX) == 0)
1232                 saltlen = RSA_PSS_SALTLEN_MAX;
1233             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO) == 0)
1234                 saltlen = RSA_PSS_SALTLEN_AUTO;
1235             else
1236                 saltlen = atoi(p->data);
1237             break;
1238         default:
1239             return 0;
1240         }
1241
1242         /*
1243          * RSA_PSS_SALTLEN_MAX seems curiously named in this check.
1244          * Contrary to what it's name suggests, it's the currently
1245          * lowest saltlen number possible.
1246          */
1247         if (saltlen < RSA_PSS_SALTLEN_MAX) {
1248             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
1249             return 0;
1250         }
1251
1252         if (rsa_pss_restricted(prsactx)) {
1253             switch (saltlen) {
1254             case RSA_PSS_SALTLEN_AUTO:
1255                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
1256                     ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
1257                     return 0;
1258                 }
1259                 break;
1260             case RSA_PSS_SALTLEN_DIGEST:
1261                 if (prsactx->min_saltlen > EVP_MD_size(prsactx->md)) {
1262                     ERR_raise_data(ERR_LIB_PROV,
1263                                    PROV_R_PSS_SALTLEN_TOO_SMALL,
1264                                    "Should be more than %d, but would be "
1265                                    "set to match digest size (%d)",
1266                                    prsactx->min_saltlen,
1267                                    EVP_MD_size(prsactx->md));
1268                     return 0;
1269                 }
1270                 break;
1271             default:
1272                 if (saltlen >= 0 && saltlen < prsactx->min_saltlen) {
1273                     ERR_raise_data(ERR_LIB_PROV,
1274                                    PROV_R_PSS_SALTLEN_TOO_SMALL,
1275                                    "Should be more than %d, "
1276                                    "but would be set to %d",
1277                                    prsactx->min_saltlen, saltlen);
1278                     return 0;
1279                 }
1280             }
1281         }
1282     }
1283
1284     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_MGF1_DIGEST);
1285     if (p != NULL) {
1286         const OSSL_PARAM *propsp =
1287             OSSL_PARAM_locate_const(params,
1288                                     OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES);
1289
1290         pmgf1mdname = mgf1mdname;
1291         if (!OSSL_PARAM_get_utf8_string(p, &pmgf1mdname, sizeof(mgf1mdname)))
1292             return 0;
1293
1294         if (propsp != NULL) {
1295             pmgf1mdprops = mgf1mdprops;
1296             if (!OSSL_PARAM_get_utf8_string(propsp,
1297                                             &pmgf1mdprops, sizeof(mgf1mdprops)))
1298                 return 0;
1299         }
1300
1301         if (pad_mode != RSA_PKCS1_PSS_PADDING) {
1302             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MGF1_MD);
1303             return  0;
1304         }
1305     }
1306
1307     prsactx->saltlen = saltlen;
1308     prsactx->pad_mode = pad_mode;
1309
1310     if (prsactx->md == NULL && pmdname == NULL
1311         && pad_mode == RSA_PKCS1_PSS_PADDING)
1312         pmdname = RSA_DEFAULT_DIGEST_NAME;
1313
1314     if (pmgf1mdname != NULL
1315         && !rsa_setup_mgf1_md(prsactx, pmgf1mdname, pmgf1mdprops))
1316         return 0;
1317
1318     if (pmdname != NULL) {
1319         if (!rsa_setup_md(prsactx, pmdname, pmdprops))
1320             return 0;
1321     } else {
1322         if (!rsa_check_padding(prsactx, NULL, NULL, prsactx->mdnid))
1323             return 0;
1324     }
1325     return 1;
1326 }
1327
1328 static const OSSL_PARAM known_settable_ctx_params[] = {
1329     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PAD_MODE, NULL, 0),
1330     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0),
1331     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PROPERTIES, NULL, 0),
1332     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_DIGEST, NULL, 0),
1333     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES, NULL, 0),
1334     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PSS_SALTLEN, NULL, 0),
1335     OSSL_PARAM_END
1336 };
1337
1338 static const OSSL_PARAM *rsa_settable_ctx_params(ossl_unused void *provctx)
1339 {
1340     /*
1341      * TODO(3.0): Should this function return a different set of settable ctx
1342      * params if the ctx is being used for a DigestSign/DigestVerify? In that
1343      * case it is not allowed to set the digest size/digest name because the
1344      * digest is explicitly set as part of the init.
1345      */
1346     return known_settable_ctx_params;
1347 }
1348
1349 static int rsa_get_ctx_md_params(void *vprsactx, OSSL_PARAM *params)
1350 {
1351     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1352
1353     if (prsactx->mdctx == NULL)
1354         return 0;
1355
1356     return EVP_MD_CTX_get_params(prsactx->mdctx, params);
1357 }
1358
1359 static const OSSL_PARAM *rsa_gettable_ctx_md_params(void *vprsactx)
1360 {
1361     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1362
1363     if (prsactx->md == NULL)
1364         return 0;
1365
1366     return EVP_MD_gettable_ctx_params(prsactx->md);
1367 }
1368
1369 static int rsa_set_ctx_md_params(void *vprsactx, const OSSL_PARAM params[])
1370 {
1371     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1372
1373     if (prsactx->mdctx == NULL)
1374         return 0;
1375
1376     return EVP_MD_CTX_set_params(prsactx->mdctx, params);
1377 }
1378
1379 static const OSSL_PARAM *rsa_settable_ctx_md_params(void *vprsactx)
1380 {
1381     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1382
1383     if (prsactx->md == NULL)
1384         return 0;
1385
1386     return EVP_MD_settable_ctx_params(prsactx->md);
1387 }
1388
1389 const OSSL_DISPATCH ossl_rsa_signature_functions[] = {
1390     { OSSL_FUNC_SIGNATURE_NEWCTX, (void (*)(void))rsa_newctx },
1391     { OSSL_FUNC_SIGNATURE_SIGN_INIT, (void (*)(void))rsa_sign_init },
1392     { OSSL_FUNC_SIGNATURE_SIGN, (void (*)(void))rsa_sign },
1393     { OSSL_FUNC_SIGNATURE_VERIFY_INIT, (void (*)(void))rsa_verify_init },
1394     { OSSL_FUNC_SIGNATURE_VERIFY, (void (*)(void))rsa_verify },
1395     { OSSL_FUNC_SIGNATURE_VERIFY_RECOVER_INIT,
1396       (void (*)(void))rsa_verify_recover_init },
1397     { OSSL_FUNC_SIGNATURE_VERIFY_RECOVER,
1398       (void (*)(void))rsa_verify_recover },
1399     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_INIT,
1400       (void (*)(void))rsa_digest_sign_init },
1401     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_UPDATE,
1402       (void (*)(void))rsa_digest_signverify_update },
1403     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_FINAL,
1404       (void (*)(void))rsa_digest_sign_final },
1405     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_INIT,
1406       (void (*)(void))rsa_digest_verify_init },
1407     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_UPDATE,
1408       (void (*)(void))rsa_digest_signverify_update },
1409     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_FINAL,
1410       (void (*)(void))rsa_digest_verify_final },
1411     { OSSL_FUNC_SIGNATURE_FREECTX, (void (*)(void))rsa_freectx },
1412     { OSSL_FUNC_SIGNATURE_DUPCTX, (void (*)(void))rsa_dupctx },
1413     { OSSL_FUNC_SIGNATURE_GET_CTX_PARAMS, (void (*)(void))rsa_get_ctx_params },
1414     { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_PARAMS,
1415       (void (*)(void))rsa_gettable_ctx_params },
1416     { OSSL_FUNC_SIGNATURE_SET_CTX_PARAMS, (void (*)(void))rsa_set_ctx_params },
1417     { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_PARAMS,
1418       (void (*)(void))rsa_settable_ctx_params },
1419     { OSSL_FUNC_SIGNATURE_GET_CTX_MD_PARAMS,
1420       (void (*)(void))rsa_get_ctx_md_params },
1421     { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_MD_PARAMS,
1422       (void (*)(void))rsa_gettable_ctx_md_params },
1423     { OSSL_FUNC_SIGNATURE_SET_CTX_MD_PARAMS,
1424       (void (*)(void))rsa_set_ctx_md_params },
1425     { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS,
1426       (void (*)(void))rsa_settable_ctx_md_params },
1427     { 0, NULL }
1428 };