providers/implementations/signature/rsa.c

   1 /*
   2  * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 /*
  11  * RSA low level APIs are deprecated for public use, but still ok for
  12  * internal use.
  13  */
  14 #include "internal/deprecated.h"
  15
  16 #include <string.h>
  17 #include <openssl/crypto.h>
  18 #include <openssl/core_dispatch.h>
  19 #include <openssl/core_names.h>
  20 #include <openssl/err.h>
  21 #include <openssl/rsa.h>
  22 #include <openssl/params.h>
  23 #include <openssl/evp.h>
  24 #include "internal/cryptlib.h"
  25 #include "internal/nelem.h"
  26 #include "internal/sizes.h"
  27 #include "crypto/rsa.h"
  28 #include "prov/providercommon.h"
  29 #include "prov/providercommonerr.h"
  30 #include "prov/implementations.h"
  31 #include "prov/provider_ctx.h"
  32 #include "prov/der_rsa.h"
  33 #include "prov/securitycheck.h"
  34
  35 #define RSA_DEFAULT_DIGEST_NAME OSSL_DIGEST_NAME_SHA1
  36
  37 static OSSL_FUNC_signature_newctx_fn rsa_newctx;
  38 static OSSL_FUNC_signature_sign_init_fn rsa_sign_init;
  39 static OSSL_FUNC_signature_verify_init_fn rsa_verify_init;
  40 static OSSL_FUNC_signature_verify_recover_init_fn rsa_verify_recover_init;
  41 static OSSL_FUNC_signature_sign_fn rsa_sign;
  42 static OSSL_FUNC_signature_verify_fn rsa_verify;
  43 static OSSL_FUNC_signature_verify_recover_fn rsa_verify_recover;
  44 static OSSL_FUNC_signature_digest_sign_init_fn rsa_digest_sign_init;
  45 static OSSL_FUNC_signature_digest_sign_update_fn rsa_digest_signverify_update;
  46 static OSSL_FUNC_signature_digest_sign_final_fn rsa_digest_sign_final;
  47 static OSSL_FUNC_signature_digest_verify_init_fn rsa_digest_verify_init;
  48 static OSSL_FUNC_signature_digest_verify_update_fn rsa_digest_signverify_update;
  49 static OSSL_FUNC_signature_digest_verify_final_fn rsa_digest_verify_final;
  50 static OSSL_FUNC_signature_freectx_fn rsa_freectx;
  51 static OSSL_FUNC_signature_dupctx_fn rsa_dupctx;
  52 static OSSL_FUNC_signature_get_ctx_params_fn rsa_get_ctx_params;
  53 static OSSL_FUNC_signature_gettable_ctx_params_fn rsa_gettable_ctx_params;
  54 static OSSL_FUNC_signature_set_ctx_params_fn rsa_set_ctx_params;
  55 static OSSL_FUNC_signature_settable_ctx_params_fn rsa_settable_ctx_params;
  56 static OSSL_FUNC_signature_get_ctx_md_params_fn rsa_get_ctx_md_params;
  57 static OSSL_FUNC_signature_gettable_ctx_md_params_fn rsa_gettable_ctx_md_params;
  58 static OSSL_FUNC_signature_set_ctx_md_params_fn rsa_set_ctx_md_params;
  59 static OSSL_FUNC_signature_settable_ctx_md_params_fn rsa_settable_ctx_md_params;
  60
  61 static OSSL_ITEM padding_item[] = {
  62     { RSA_PKCS1_PADDING,        OSSL_PKEY_RSA_PAD_MODE_PKCSV15 },
  63     { RSA_SSLV23_PADDING,       OSSL_PKEY_RSA_PAD_MODE_SSLV23 },
  64     { RSA_NO_PADDING,           OSSL_PKEY_RSA_PAD_MODE_NONE },
  65     { RSA_X931_PADDING,         OSSL_PKEY_RSA_PAD_MODE_X931 },
  66     { RSA_PKCS1_PSS_PADDING,    OSSL_PKEY_RSA_PAD_MODE_PSS },
  67     { 0,                        NULL     }
  68 };
  69
  70 /*
  71  * What's passed as an actual key is defined by the KEYMGMT interface.
  72  * We happen to know that our KEYMGMT simply passes RSA structures, so
  73  * we use that here too.
  74  */
  75
  76 typedef struct {
  77     OPENSSL_CTX *libctx;
  78     char *propq;
  79     RSA *rsa;
  80     int operation;
  81
  82     /*
  83      * Flag to determine if the hash function can be changed (1) or not (0)
  84      * Because it's dangerous to change during a DigestSign or DigestVerify
  85      * operation, this flag is cleared by their Init function, and set again
  86      * by their Final function.
  87      */
  88     unsigned int flag_allow_md : 1;
  89
  90     /* The Algorithm Identifier of the combined signature algorithm */
  91     unsigned char aid_buf[128];
  92     unsigned char *aid;
  93     size_t  aid_len;
  94
  95     /* main digest */
  96     EVP_MD *md;
  97     EVP_MD_CTX *mdctx;
  98     int mdnid;
  99     char mdname[OSSL_MAX_NAME_SIZE]; /* Purely informational */
 100
 101     /* RSA padding mode */
 102     int pad_mode;
 103     /* message digest for MGF1 */
 104     EVP_MD *mgf1_md;
 105     char mgf1_mdname[OSSL_MAX_NAME_SIZE]; /* Purely informational */
 106     /* PSS salt length */
 107     int saltlen;
 108     /* Minimum salt length or -1 if no PSS parameter restriction */
 109     int min_saltlen;
 110
 111     /* Temp buffer */
 112     unsigned char *tbuf;
 113
 114 } PROV_RSA_CTX;
 115
 116 static size_t rsa_get_md_size(const PROV_RSA_CTX *prsactx)
 117 {
 118     if (prsactx->md != NULL)
 119         return EVP_MD_size(prsactx->md);
 120     return 0;
 121 }
 122
 123 static int rsa_check_padding(int mdnid, int padding)
 124 {
 125     if (padding == RSA_NO_PADDING) {
 126         ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE);
 127         return 0;
 128     }
 129
 130     if (padding == RSA_X931_PADDING) {
 131         if (RSA_X931_hash_id(mdnid) == -1) {
 132             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_X931_DIGEST);
 133             return 0;
 134         }
 135     }
 136
 137     return 1;
 138 }
 139
 140 static int rsa_check_parameters(PROV_RSA_CTX *prsactx)
 141 {
 142     if (prsactx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 143         int max_saltlen;
 144
 145         /* See if minimum salt length exceeds maximum possible */
 146         max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(prsactx->md);
 147         if ((RSA_bits(prsactx->rsa) & 0x7) == 1)
 148             max_saltlen--;
 149         if (prsactx->min_saltlen < 0 || prsactx->min_saltlen > max_saltlen) {
 150             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
 151             return 0;
 152         }
 153     }
 154     return 1;
 155 }
 156
 157 static void *rsa_newctx(void *provctx, const char *propq)
 158 {
 159     PROV_RSA_CTX *prsactx = NULL;
 160     char *propq_copy = NULL;
 161
 162     if (!ossl_prov_is_running())
 163         return NULL;
 164
 165     if ((prsactx = OPENSSL_zalloc(sizeof(PROV_RSA_CTX))) == NULL
 166         || (propq != NULL
 167             && (propq_copy = OPENSSL_strdup(propq)) == NULL)) {
 168         OPENSSL_free(prsactx);
 169         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 170         return NULL;
 171     }
 172
 173     prsactx->libctx = PROV_LIBRARY_CONTEXT_OF(provctx);
 174     prsactx->flag_allow_md = 1;
 175     prsactx->propq = propq_copy;
 176     return prsactx;
 177 }
 178
 179 /* True if PSS parameters are restricted */
 180 #define rsa_pss_restricted(prsactx) (prsactx->min_saltlen != -1)
 181
 182 static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
 183                         const char *mdprops)
 184 {
 185     if (mdprops == NULL)
 186         mdprops = ctx->propq;
 187
 188     if (mdname != NULL) {
 189         WPACKET pkt;
 190         EVP_MD *md = EVP_MD_fetch(ctx->libctx, mdname, mdprops);
 191         int sha1_allowed = (ctx->operation != EVP_PKEY_OP_SIGN);
 192         int md_nid = digest_rsa_sign_get_md_nid(md, sha1_allowed);
 193         size_t mdname_len = strlen(mdname);
 194
 195         if (md == NULL
 196             || md_nid == NID_undef
 197             || !rsa_check_padding(md_nid, ctx->pad_mode)
 198             || mdname_len >= sizeof(ctx->mdname)) {
 199             if (md == NULL)
 200                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 201                                "%s could not be fetched", mdname);
 202             if (md_nid == NID_undef)
 203                 ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
 204                                "digest=%s", mdname);
 205             if (mdname_len >= sizeof(ctx->mdname))
 206                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 207                                "%s exceeds name buffer length", mdname);
 208             EVP_MD_free(md);
 209             return 0;
 210         }
 211
 212         EVP_MD_CTX_free(ctx->mdctx);
 213         EVP_MD_free(ctx->md);
 214
 215         /*
 216          * TODO(3.0) Should we care about DER writing errors?
 217          * All it really means is that for some reason, there's no
 218          * AlgorithmIdentifier to be had (consider RSA with MD5-SHA1),
 219          * but the operation itself is still valid, just as long as it's
 220          * not used to construct anything that needs an AlgorithmIdentifier.
 221          */
 222         ctx->aid_len = 0;
 223         if (WPACKET_init_der(&pkt, ctx->aid_buf, sizeof(ctx->aid_buf))
 224             && ossl_DER_w_algorithmIdentifier_MDWithRSAEncryption(&pkt, -1,
 225                                                                   ctx->rsa,
 226                                                                   md_nid)
 227             && WPACKET_finish(&pkt)) {
 228             WPACKET_get_total_written(&pkt, &ctx->aid_len);
 229             ctx->aid = WPACKET_get_curr(&pkt);
 230         }
 231         WPACKET_cleanup(&pkt);
 232
 233         ctx->mdctx = NULL;
 234         ctx->md = md;
 235         ctx->mdnid = md_nid;
 236         OPENSSL_strlcpy(ctx->mdname, mdname, sizeof(ctx->mdname));
 237     }
 238
 239     return 1;
 240 }
 241
 242 static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
 243                              const char *mdprops)
 244 {
 245     size_t len;
 246     EVP_MD *md = NULL;
 247
 248     if (mdprops == NULL)
 249         mdprops = ctx->propq;
 250
 251     if (ctx->mgf1_mdname[0] != '\0')
 252         EVP_MD_free(ctx->mgf1_md);
 253
 254     if ((md = EVP_MD_fetch(ctx->libctx, mdname, mdprops)) == NULL) {
 255         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 256                        "%s could not be fetched", mdname);
 257         return 0;
 258     }
 259     /* The default for mgf1 is SHA1 - so allow SHA1 */
 260     if (digest_rsa_sign_get_md_nid(md, 1) == NID_undef) {
 261         ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
 262                        "digest=%s", mdname);
 263         EVP_MD_free(md);
 264         return 0;
 265     }
 266     ctx->mgf1_md = md;
 267     len = OPENSSL_strlcpy(ctx->mgf1_mdname, mdname, sizeof(ctx->mgf1_mdname));
 268     if (len >= sizeof(ctx->mgf1_mdname)) {
 269         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 270                        "%s exceeds name buffer length", mdname);
 271         return 0;
 272     }
 273
 274     return 1;
 275 }
 276
 277 static int rsa_signverify_init(void *vprsactx, void *vrsa, int operation)
 278 {
 279     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 280
 281     if (!ossl_prov_is_running())
 282         return 0;
 283
 284     if (prsactx == NULL || vrsa == NULL || !RSA_up_ref(vrsa))
 285         return 0;
 286
 287     RSA_free(prsactx->rsa);
 288     prsactx->rsa = vrsa;
 289     prsactx->operation = operation;
 290
 291     if (!rsa_check_key(vrsa, operation == EVP_PKEY_OP_SIGN)) {
 292         ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH);
 293         return 0;
 294     }
 295
 296     /* Maximum for sign, auto for verify */
 297     prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
 298     prsactx->min_saltlen = -1;
 299
 300     switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
 301     case RSA_FLAG_TYPE_RSA:
 302         prsactx->pad_mode = RSA_PKCS1_PADDING;
 303         break;
 304     case RSA_FLAG_TYPE_RSASSAPSS:
 305         prsactx->pad_mode = RSA_PKCS1_PSS_PADDING;
 306
 307         {
 308             const RSA_PSS_PARAMS_30 *pss =
 309                 rsa_get0_pss_params_30(prsactx->rsa);
 310
 311             if (!rsa_pss_params_30_is_unrestricted(pss)) {
 312                 int md_nid = rsa_pss_params_30_hashalg(pss);
 313                 int mgf1md_nid = rsa_pss_params_30_maskgenhashalg(pss);
 314                 int min_saltlen = rsa_pss_params_30_saltlen(pss);
 315                 const char *mdname, *mgf1mdname;
 316                 size_t len;
 317
 318                 mdname = rsa_oaeppss_nid2name(md_nid);
 319                 mgf1mdname = rsa_oaeppss_nid2name(mgf1md_nid);
 320                 prsactx->min_saltlen = min_saltlen;
 321
 322                 if (mdname == NULL) {
 323                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 324                                    "PSS restrictions lack hash algorithm");
 325                     return 0;
 326                 }
 327                 if (mgf1mdname == NULL) {
 328                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 329                                    "PSS restrictions lack MGF1 hash algorithm");
 330                     return 0;
 331                 }
 332
 333                 len = OPENSSL_strlcpy(prsactx->mdname, mdname,
 334                                       sizeof(prsactx->mdname));
 335                 if (len >= sizeof(prsactx->mdname)) {
 336                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 337                                    "hash algorithm name too long");
 338                     return 0;
 339                 }
 340                 len = OPENSSL_strlcpy(prsactx->mgf1_mdname, mgf1mdname,
 341                                       sizeof(prsactx->mgf1_mdname));
 342                 if (len >= sizeof(prsactx->mgf1_mdname)) {
 343                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
 344                                    "MGF1 hash algorithm name too long");
 345                     return 0;
 346                 }
 347                 prsactx->saltlen = min_saltlen;
 348
 349                 return rsa_setup_md(prsactx, mdname, prsactx->propq)
 350                     && rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq)
 351                     && rsa_check_parameters(prsactx);
 352             }
 353         }
 354
 355         break;
 356     default:
 357         ERR_raise(ERR_LIB_RSA, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
 358         return 0;
 359     }
 360
 361     return 1;
 362 }
 363
 364 static int setup_tbuf(PROV_RSA_CTX *ctx)
 365 {
 366     if (ctx->tbuf != NULL)
 367         return 1;
 368     if ((ctx->tbuf = OPENSSL_malloc(RSA_size(ctx->rsa))) == NULL) {
 369         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 370         return 0;
 371     }
 372     return 1;
 373 }
 374
 375 static void clean_tbuf(PROV_RSA_CTX *ctx)
 376 {
 377     if (ctx->tbuf != NULL)
 378         OPENSSL_cleanse(ctx->tbuf, RSA_size(ctx->rsa));
 379 }
 380
 381 static void free_tbuf(PROV_RSA_CTX *ctx)
 382 {
 383     clean_tbuf(ctx);
 384     OPENSSL_free(ctx->tbuf);
 385     ctx->tbuf = NULL;
 386 }
 387
 388 static int rsa_sign_init(void *vprsactx, void *vrsa)
 389 {
 390     if (!ossl_prov_is_running())
 391         return 0;
 392     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_SIGN);
 393 }
 394
 395 static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen,
 396                     size_t sigsize, const unsigned char *tbs, size_t tbslen)
 397 {
 398     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 399     int ret;
 400     size_t rsasize = RSA_size(prsactx->rsa);
 401     size_t mdsize = rsa_get_md_size(prsactx);
 402
 403     if (!ossl_prov_is_running())
 404         return 0;
 405
 406     if (sig == NULL) {
 407         *siglen = rsasize;
 408         return 1;
 409     }
 410
 411     if (sigsize < rsasize) {
 412         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SIGNATURE_SIZE,
 413                        "is %zu, should be at least %zu", sigsize, rsasize);
 414         return 0;
 415     }
 416
 417     if (mdsize != 0) {
 418         if (tbslen != mdsize) {
 419             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH);
 420             return 0;
 421         }
 422
 423 #ifndef FIPS_MODULE
 424         if (EVP_MD_is_a(prsactx->md, OSSL_DIGEST_NAME_MDC2)) {
 425             unsigned int sltmp;
 426
 427             if (prsactx->pad_mode != RSA_PKCS1_PADDING) {
 428                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 429                                "only PKCS#1 padding supported with MDC2");
 430                 return 0;
 431             }
 432             ret = RSA_sign_ASN1_OCTET_STRING(0, tbs, tbslen, sig, &sltmp,
 433                                              prsactx->rsa);
 434
 435             if (ret <= 0) {
 436                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 437                 return 0;
 438             }
 439             ret = sltmp;
 440             goto end;
 441         }
 442 #endif
 443         switch (prsactx->pad_mode) {
 444         case RSA_X931_PADDING:
 445             if ((size_t)RSA_size(prsactx->rsa) < tbslen + 1) {
 446                 ERR_raise_data(ERR_LIB_PROV, PROV_R_KEY_SIZE_TOO_SMALL,
 447                                "RSA key size = %d, expected minimum = %d",
 448                                RSA_size(prsactx->rsa), tbslen + 1);
 449                 return 0;
 450             }
 451             if (!setup_tbuf(prsactx)) {
 452                 ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 453                 return 0;
 454             }
 455             memcpy(prsactx->tbuf, tbs, tbslen);
 456             prsactx->tbuf[tbslen] = RSA_X931_hash_id(prsactx->mdnid);
 457             ret = RSA_private_encrypt(tbslen + 1, prsactx->tbuf,
 458                                       sig, prsactx->rsa, RSA_X931_PADDING);
 459             clean_tbuf(prsactx);
 460             break;
 461
 462         case RSA_PKCS1_PADDING:
 463             {
 464                 unsigned int sltmp;
 465
 466                 ret = RSA_sign(prsactx->mdnid, tbs, tbslen, sig, &sltmp,
 467                                prsactx->rsa);
 468                 if (ret <= 0) {
 469                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 470                     return 0;
 471                 }
 472                 ret = sltmp;
 473             }
 474             break;
 475
 476         case RSA_PKCS1_PSS_PADDING:
 477             /* Check PSS restrictions */
 478             if (rsa_pss_restricted(prsactx)) {
 479                 switch (prsactx->saltlen) {
 480                 case RSA_PSS_SALTLEN_DIGEST:
 481                     if (prsactx->min_saltlen > EVP_MD_size(prsactx->md)) {
 482                         ERR_raise_data(ERR_LIB_PROV,
 483                                        PROV_R_PSS_SALTLEN_TOO_SMALL,
 484                                        "minimum salt length set to %d, "
 485                                        "but the digest only gives %d",
 486                                        prsactx->min_saltlen,
 487                                        EVP_MD_size(prsactx->md));
 488                         return 0;
 489                     }
 490                     /* FALLTHRU */
 491                 default:
 492                     if (prsactx->saltlen >= 0
 493                         && prsactx->saltlen < prsactx->min_saltlen) {
 494                         ERR_raise_data(ERR_LIB_PROV,
 495                                        PROV_R_PSS_SALTLEN_TOO_SMALL,
 496                                        "minimum salt length set to %d, but the"
 497                                        "actual salt length is only set to %d",
 498                                        prsactx->min_saltlen,
 499                                        prsactx->saltlen);
 500                         return 0;
 501                     }
 502                     break;
 503                 }
 504             }
 505             if (!setup_tbuf(prsactx))
 506                 return 0;
 507             if (!RSA_padding_add_PKCS1_PSS_mgf1(prsactx->rsa,
 508                                                 prsactx->tbuf, tbs,
 509                                                 prsactx->md, prsactx->mgf1_md,
 510                                                 prsactx->saltlen)) {
 511                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 512                 return 0;
 513             }
 514             ret = RSA_private_encrypt(RSA_size(prsactx->rsa), prsactx->tbuf,
 515                                       sig, prsactx->rsa, RSA_NO_PADDING);
 516             clean_tbuf(prsactx);
 517             break;
 518
 519         default:
 520             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 521                            "Only X.931, PKCS#1 v1.5 or PSS padding allowed");
 522             return 0;
 523         }
 524     } else {
 525         ret = RSA_private_encrypt(tbslen, tbs, sig, prsactx->rsa,
 526                                   prsactx->pad_mode);
 527     }
 528
 529 #ifndef FIPS_MODULE
 530  end:
 531 #endif
 532     if (ret <= 0) {
 533         ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 534         return 0;
 535     }
 536
 537     *siglen = ret;
 538     return 1;
 539 }
 540
 541 static int rsa_verify_recover_init(void *vprsactx, void *vrsa)
 542 {
 543     if (!ossl_prov_is_running())
 544         return 0;
 545     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFYRECOVER);
 546 }
 547
 548 static int rsa_verify_recover(void *vprsactx,
 549                               unsigned char *rout,
 550                               size_t *routlen,
 551                               size_t routsize,
 552                               const unsigned char *sig,
 553                               size_t siglen)
 554 {
 555     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 556     int ret;
 557
 558     if (!ossl_prov_is_running())
 559         return 0;
 560
 561     if (rout == NULL) {
 562         *routlen = RSA_size(prsactx->rsa);
 563         return 1;
 564     }
 565
 566     if (prsactx->md != NULL) {
 567         switch (prsactx->pad_mode) {
 568         case RSA_X931_PADDING:
 569             if (!setup_tbuf(prsactx))
 570                 return 0;
 571             ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
 572                                      RSA_X931_PADDING);
 573             if (ret < 1) {
 574                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 575                 return 0;
 576             }
 577             ret--;
 578             if (prsactx->tbuf[ret] != RSA_X931_hash_id(prsactx->mdnid)) {
 579                 ERR_raise(ERR_LIB_PROV, PROV_R_ALGORITHM_MISMATCH);
 580                 return 0;
 581             }
 582             if (ret != EVP_MD_size(prsactx->md)) {
 583                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH,
 584                                "Should be %d, but got %d",
 585                                EVP_MD_size(prsactx->md), ret);
 586                 return 0;
 587             }
 588
 589             *routlen = ret;
 590             if (rout != prsactx->tbuf) {
 591                 if (routsize < (size_t)ret) {
 592                     ERR_raise_data(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL,
 593                                    "buffer size is %d, should be %d",
 594                                    routsize, ret);
 595                     return 0;
 596                 }
 597                 memcpy(rout, prsactx->tbuf, ret);
 598             }
 599             break;
 600
 601         case RSA_PKCS1_PADDING:
 602             {
 603                 size_t sltmp;
 604
 605                 ret = int_rsa_verify(prsactx->mdnid, NULL, 0, rout, &sltmp,
 606                                      sig, siglen, prsactx->rsa);
 607                 if (ret <= 0) {
 608                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 609                     return 0;
 610                 }
 611                 ret = sltmp;
 612             }
 613             break;
 614
 615         default:
 616             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 617                            "Only X.931 or PKCS#1 v1.5 padding allowed");
 618             return 0;
 619         }
 620     } else {
 621         ret = RSA_public_decrypt(siglen, sig, rout, prsactx->rsa,
 622                                  prsactx->pad_mode);
 623         if (ret < 0) {
 624             ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 625             return 0;
 626         }
 627     }
 628     *routlen = ret;
 629     return 1;
 630 }
 631
 632 static int rsa_verify_init(void *vprsactx, void *vrsa)
 633 {
 634     if (!ossl_prov_is_running())
 635         return 0;
 636     return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFY);
 637 }
 638
 639 static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
 640                       const unsigned char *tbs, size_t tbslen)
 641 {
 642     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 643     size_t rslen;
 644
 645     if (!ossl_prov_is_running())
 646         return 0;
 647     if (prsactx->md != NULL) {
 648         switch (prsactx->pad_mode) {
 649         case RSA_PKCS1_PADDING:
 650             if (!RSA_verify(prsactx->mdnid, tbs, tbslen, sig, siglen,
 651                             prsactx->rsa)) {
 652                 ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 653                 return 0;
 654             }
 655             return 1;
 656         case RSA_X931_PADDING:
 657             if (!setup_tbuf(prsactx))
 658                 return 0;
 659             if (rsa_verify_recover(prsactx, prsactx->tbuf, &rslen, 0,
 660                                    sig, siglen) <= 0)
 661                 return 0;
 662             break;
 663         case RSA_PKCS1_PSS_PADDING:
 664             {
 665                 int ret;
 666                 size_t mdsize;
 667
 668                 /*
 669                  * We need to check this for the RSA_verify_PKCS1_PSS_mgf1()
 670                  * call
 671                  */
 672                 mdsize = rsa_get_md_size(prsactx);
 673                 if (tbslen != mdsize) {
 674                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH,
 675                                    "Should be %d, but got %d",
 676                                    mdsize, tbslen);
 677                     return 0;
 678                 }
 679
 680                 if (!setup_tbuf(prsactx))
 681                     return 0;
 682                 ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf,
 683                                          prsactx->rsa, RSA_NO_PADDING);
 684                 if (ret <= 0) {
 685                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 686                     return 0;
 687                 }
 688                 ret = RSA_verify_PKCS1_PSS_mgf1(prsactx->rsa, tbs,
 689                                                 prsactx->md, prsactx->mgf1_md,
 690                                                 prsactx->tbuf,
 691                                                 prsactx->saltlen);
 692                 if (ret <= 0) {
 693                     ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 694                     return 0;
 695                 }
 696                 return 1;
 697             }
 698         default:
 699             ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE,
 700                            "Only X.931, PKCS#1 v1.5 or PSS padding allowed");
 701             return 0;
 702         }
 703     } else {
 704         if (!setup_tbuf(prsactx))
 705             return 0;
 706         rslen = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
 707                                    prsactx->pad_mode);
 708         if (rslen == 0) {
 709             ERR_raise(ERR_LIB_PROV, ERR_LIB_RSA);
 710             return 0;
 711         }
 712     }
 713
 714     if ((rslen != tbslen) || memcmp(tbs, prsactx->tbuf, rslen))
 715         return 0;
 716
 717     return 1;
 718 }
 719
 720 static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
 721                                       void *vrsa, int operation)
 722 {
 723     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 724
 725     if (!ossl_prov_is_running())
 726         return 0;
 727
 728     if (prsactx != NULL)
 729         prsactx->flag_allow_md = 0;
 730     if (!rsa_signverify_init(vprsactx, vrsa, operation)
 731         || !rsa_setup_md(prsactx, mdname, NULL)) /* TODO RL */
 732         return 0;
 733
 734     prsactx->mdctx = EVP_MD_CTX_new();
 735     if (prsactx->mdctx == NULL) {
 736         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 737         goto error;
 738     }
 739
 740     if (!EVP_DigestInit_ex(prsactx->mdctx, prsactx->md, NULL))
 741         goto error;
 742
 743     return 1;
 744
 745  error:
 746     EVP_MD_CTX_free(prsactx->mdctx);
 747     EVP_MD_free(prsactx->md);
 748     prsactx->mdctx = NULL;
 749     prsactx->md = NULL;
 750     return 0;
 751 }
 752
 753 static int rsa_digest_signverify_update(void *vprsactx,
 754                                         const unsigned char *data,
 755                                         size_t datalen)
 756 {
 757     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 758
 759     if (prsactx == NULL || prsactx->mdctx == NULL)
 760         return 0;
 761
 762     return EVP_DigestUpdate(prsactx->mdctx, data, datalen);
 763 }
 764
 765 static int rsa_digest_sign_init(void *vprsactx, const char *mdname,
 766                                 void *vrsa)
 767 {
 768     if (!ossl_prov_is_running())
 769         return 0;
 770     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
 771                                       EVP_PKEY_OP_SIGN);
 772 }
 773
 774 static int rsa_digest_sign_final(void *vprsactx, unsigned char *sig,
 775                                  size_t *siglen, size_t sigsize)
 776 {
 777     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 778     unsigned char digest[EVP_MAX_MD_SIZE];
 779     unsigned int dlen = 0;
 780
 781     if (!ossl_prov_is_running() || prsactx == NULL)
 782         return 0;
 783     prsactx->flag_allow_md = 1;
 784     if (prsactx->mdctx == NULL)
 785         return 0;
 786     /*
 787      * If sig is NULL then we're just finding out the sig size. Other fields
 788      * are ignored. Defer to rsa_sign.
 789      */
 790     if (sig != NULL) {
 791         /*
 792          * The digests used here are all known (see rsa_get_md_nid()), so they
 793          * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
 794          */
 795         if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
 796             return 0;
 797     }
 798
 799     return rsa_sign(vprsactx, sig, siglen, sigsize, digest, (size_t)dlen);
 800 }
 801
 802 static int rsa_digest_verify_init(void *vprsactx, const char *mdname,
 803                                   void *vrsa)
 804 {
 805     if (!ossl_prov_is_running())
 806         return 0;
 807     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
 808                                       EVP_PKEY_OP_VERIFY);
 809 }
 810
 811 int rsa_digest_verify_final(void *vprsactx, const unsigned char *sig,
 812                             size_t siglen)
 813 {
 814     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 815     unsigned char digest[EVP_MAX_MD_SIZE];
 816     unsigned int dlen = 0;
 817
 818     if (!ossl_prov_is_running())
 819         return 0;
 820
 821     if (prsactx == NULL)
 822         return 0;
 823     prsactx->flag_allow_md = 1;
 824     if (prsactx->mdctx == NULL)
 825         return 0;
 826
 827     /*
 828      * The digests used here are all known (see rsa_get_md_nid()), so they
 829      * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
 830      */
 831     if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
 832         return 0;
 833
 834     return rsa_verify(vprsactx, sig, siglen, digest, (size_t)dlen);
 835 }
 836
 837 static void rsa_freectx(void *vprsactx)
 838 {
 839     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 840
 841     if (prsactx == NULL)
 842         return;
 843
 844     EVP_MD_CTX_free(prsactx->mdctx);
 845     EVP_MD_free(prsactx->md);
 846     EVP_MD_free(prsactx->mgf1_md);
 847     OPENSSL_free(prsactx->propq);
 848     free_tbuf(prsactx);
 849     RSA_free(prsactx->rsa);
 850
 851     OPENSSL_clear_free(prsactx, sizeof(*prsactx));
 852 }
 853
 854 static void *rsa_dupctx(void *vprsactx)
 855 {
 856     PROV_RSA_CTX *srcctx = (PROV_RSA_CTX *)vprsactx;
 857     PROV_RSA_CTX *dstctx;
 858
 859     if (!ossl_prov_is_running())
 860         return NULL;
 861
 862     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
 863     if (dstctx == NULL) {
 864         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
 865         return NULL;
 866     }
 867
 868     *dstctx = *srcctx;
 869     dstctx->rsa = NULL;
 870     dstctx->md = NULL;
 871     dstctx->mdctx = NULL;
 872     dstctx->tbuf = NULL;
 873
 874     if (srcctx->rsa != NULL && !RSA_up_ref(srcctx->rsa))
 875         goto err;
 876     dstctx->rsa = srcctx->rsa;
 877
 878     if (srcctx->md != NULL && !EVP_MD_up_ref(srcctx->md))
 879         goto err;
 880     dstctx->md = srcctx->md;
 881
 882     if (srcctx->mgf1_md != NULL && !EVP_MD_up_ref(srcctx->mgf1_md))
 883         goto err;
 884     dstctx->mgf1_md = srcctx->mgf1_md;
 885
 886     if (srcctx->mdctx != NULL) {
 887         dstctx->mdctx = EVP_MD_CTX_new();
 888         if (dstctx->mdctx == NULL
 889                 || !EVP_MD_CTX_copy_ex(dstctx->mdctx, srcctx->mdctx))
 890             goto err;
 891     }
 892
 893     return dstctx;
 894  err:
 895     rsa_freectx(dstctx);
 896     return NULL;
 897 }
 898
 899 static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params)
 900 {
 901     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 902     OSSL_PARAM *p;
 903
 904     if (prsactx == NULL || params == NULL)
 905         return 0;
 906
 907     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_ALGORITHM_ID);
 908     if (p != NULL
 909         && !OSSL_PARAM_set_octet_string(p, prsactx->aid, prsactx->aid_len))
 910         return 0;
 911
 912     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_PAD_MODE);
 913     if (p != NULL)
 914         switch (p->data_type) {
 915         case OSSL_PARAM_INTEGER:
 916             if (!OSSL_PARAM_set_int(p, prsactx->pad_mode))
 917                 return 0;
 918             break;
 919         case OSSL_PARAM_UTF8_STRING:
 920             {
 921                 int i;
 922                 const char *word = NULL;
 923
 924                 for (i = 0; padding_item[i].id != 0; i++) {
 925                     if (prsactx->pad_mode == (int)padding_item[i].id) {
 926                         word = padding_item[i].ptr;
 927                         break;
 928                     }
 929                 }
 930
 931                 if (word != NULL) {
 932                     if (!OSSL_PARAM_set_utf8_string(p, word))
 933                         return 0;
 934                 } else {
 935                     ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
 936                 }
 937             }
 938             break;
 939         default:
 940             return 0;
 941         }
 942
 943     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_DIGEST);
 944     if (p != NULL && !OSSL_PARAM_set_utf8_string(p, prsactx->mdname))
 945         return 0;
 946
 947     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_MGF1_DIGEST);
 948     if (p != NULL && !OSSL_PARAM_set_utf8_string(p, prsactx->mgf1_mdname))
 949         return 0;
 950
 951     p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_PSS_SALTLEN);
 952     if (p != NULL) {
 953         if (p->data_type == OSSL_PARAM_INTEGER) {
 954             if (!OSSL_PARAM_set_int(p, prsactx->saltlen))
 955                 return 0;
 956         } else if (p->data_type == OSSL_PARAM_UTF8_STRING) {
 957             const char *value = NULL;
 958
 959             switch (prsactx->saltlen) {
 960             case RSA_PSS_SALTLEN_DIGEST:
 961                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST;
 962                 break;
 963             case RSA_PSS_SALTLEN_MAX:
 964                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_MAX;
 965                 break;
 966             case RSA_PSS_SALTLEN_AUTO:
 967                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO;
 968                 break;
 969             default:
 970                 {
 971                     int len = BIO_snprintf(p->data, p->data_size, "%d",
 972                                            prsactx->saltlen);
 973
 974                     if (len <= 0)
 975                         return 0;
 976                     p->return_size = len;
 977                     break;
 978                 }
 979             }
 980             if (value != NULL
 981                 && !OSSL_PARAM_set_utf8_string(p, value))
 982                 return 0;
 983         }
 984     }
 985
 986     return 1;
 987 }
 988
 989 static const OSSL_PARAM known_gettable_ctx_params[] = {
 990     OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_ALGORITHM_ID, NULL, 0),
 991     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PAD_MODE, NULL, 0),
 992     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0),
 993     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_DIGEST, NULL, 0),
 994     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PSS_SALTLEN, NULL, 0),
 995     OSSL_PARAM_END
 996 };
 997
 998 static const OSSL_PARAM *rsa_gettable_ctx_params(ossl_unused void *vctx)
 999 {
1000     return known_gettable_ctx_params;
1001 }
1002
1003 static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
1004 {
1005     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1006     const OSSL_PARAM *p;
1007
1008     if (prsactx == NULL || params == NULL)
1009         return 0;
1010
1011     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST);
1012     /* Not allowed during certain operations */
1013     if (p != NULL && !prsactx->flag_allow_md)
1014         return 0;
1015     if (p != NULL) {
1016         char mdname[OSSL_MAX_NAME_SIZE] = "", *pmdname = mdname;
1017         char mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmdprops = mdprops;
1018         const OSSL_PARAM *propsp =
1019             OSSL_PARAM_locate_const(params,
1020                                     OSSL_SIGNATURE_PARAM_PROPERTIES);
1021
1022         if (!OSSL_PARAM_get_utf8_string(p, &pmdname, sizeof(mdname)))
1023             return 0;
1024
1025         if (propsp == NULL)
1026             pmdprops = NULL;
1027         else if (!OSSL_PARAM_get_utf8_string(propsp,
1028                                              &pmdprops, sizeof(mdprops)))
1029             return 0;
1030
1031         if (rsa_pss_restricted(prsactx)) {
1032             /* TODO(3.0) figure out what to do for prsactx->md == NULL */
1033             if (prsactx->md == NULL || EVP_MD_is_a(prsactx->md, mdname))
1034                 return 1;
1035             ERR_raise(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED);
1036             return 0;
1037         }
1038
1039         /* non-PSS code follows */
1040         if (!rsa_setup_md(prsactx, mdname, pmdprops))
1041             return 0;
1042     }
1043
1044     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_PAD_MODE);
1045     if (p != NULL) {
1046         int pad_mode = 0;
1047         const char *err_extra_text = NULL;
1048
1049         switch (p->data_type) {
1050         case OSSL_PARAM_INTEGER: /* Support for legacy pad mode number */
1051             if (!OSSL_PARAM_get_int(p, &pad_mode))
1052                 return 0;
1053             break;
1054         case OSSL_PARAM_UTF8_STRING:
1055             {
1056                 int i;
1057
1058                 if (p->data == NULL)
1059                     return 0;
1060
1061                 for (i = 0; padding_item[i].id != 0; i++) {
1062                     if (strcmp(p->data, padding_item[i].ptr) == 0) {
1063                         pad_mode = padding_item[i].id;
1064                         break;
1065                     }
1066                 }
1067             }
1068             break;
1069         default:
1070             return 0;
1071         }
1072
1073         switch (pad_mode) {
1074         case RSA_PKCS1_OAEP_PADDING:
1075             /*
1076              * OAEP padding is for asymmetric cipher only so is not compatible
1077              * with signature use.
1078              */
1079             err_extra_text = "OAEP padding not allowed for signing / verifying";
1080             goto bad_pad;
1081         case RSA_PKCS1_PSS_PADDING:
1082             if ((prsactx->operation
1083                  & (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)) == 0) {
1084                 err_extra_text =
1085                     "PSS padding only allowed for sign and verify operations";
1086                 goto bad_pad;
1087             }
1088             if (prsactx->md == NULL
1089                 && !rsa_setup_md(prsactx, RSA_DEFAULT_DIGEST_NAME, NULL)) {
1090                 return 0;
1091             }
1092             break;
1093         case RSA_PKCS1_PADDING:
1094             err_extra_text = "PKCS#1 padding not allowed with RSA-PSS";
1095             goto cont;
1096         case RSA_SSLV23_PADDING:
1097             err_extra_text = "SSLv3 padding not allowed with RSA-PSS";
1098             goto cont;
1099         case RSA_NO_PADDING:
1100             err_extra_text = "No padding not allowed with RSA-PSS";
1101             goto cont;
1102         case RSA_X931_PADDING:
1103             err_extra_text = "X.931 padding not allowed with RSA-PSS";
1104         cont:
1105             if (RSA_test_flags(prsactx->rsa,
1106                                RSA_FLAG_TYPE_MASK) == RSA_FLAG_TYPE_RSA)
1107                 break;
1108             /* FALLTHRU */
1109         default:
1110         bad_pad:
1111             if (err_extra_text == NULL)
1112                 ERR_raise(ERR_LIB_PROV,
1113                           PROV_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
1114             else
1115                 ERR_raise_data(ERR_LIB_PROV,
1116                                PROV_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE,
1117                                err_extra_text);
1118             return 0;
1119         }
1120         if (!rsa_check_padding(prsactx->mdnid, pad_mode))
1121             return 0;
1122         prsactx->pad_mode = pad_mode;
1123     }
1124
1125     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_PSS_SALTLEN);
1126     if (p != NULL) {
1127         int saltlen;
1128
1129         if (prsactx->pad_mode != RSA_PKCS1_PSS_PADDING) {
1130             ERR_raise_data(ERR_LIB_PROV, PROV_R_NOT_SUPPORTED,
1131                            "PSS saltlen can only be specified if "
1132                            "PSS padding has been specified first");
1133             return 0;
1134         }
1135
1136         switch (p->data_type) {
1137         case OSSL_PARAM_INTEGER: /* Support for legacy pad mode number */
1138             if (!OSSL_PARAM_get_int(p, &saltlen))
1139                 return 0;
1140             break;
1141         case OSSL_PARAM_UTF8_STRING:
1142             if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST) == 0)
1143                 saltlen = RSA_PSS_SALTLEN_DIGEST;
1144             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_MAX) == 0)
1145                 saltlen = RSA_PSS_SALTLEN_MAX;
1146             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO) == 0)
1147                 saltlen = RSA_PSS_SALTLEN_AUTO;
1148             else
1149                 saltlen = atoi(p->data);
1150             break;
1151         default:
1152             return 0;
1153         }
1154
1155         /*
1156          * RSA_PSS_SALTLEN_MAX seems curiously named in this check.
1157          * Contrary to what it's name suggests, it's the currently
1158          * lowest saltlen number possible.
1159          */
1160         if (saltlen < RSA_PSS_SALTLEN_MAX) {
1161             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
1162             return 0;
1163         }
1164
1165         if (rsa_pss_restricted(prsactx)) {
1166             switch (saltlen) {
1167             case RSA_PSS_SALTLEN_AUTO:
1168                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
1169                     ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
1170                     return 0;
1171                 }
1172                 break;
1173             case RSA_PSS_SALTLEN_DIGEST:
1174                 if (prsactx->min_saltlen > EVP_MD_size(prsactx->md)) {
1175                     ERR_raise_data(ERR_LIB_PROV,
1176                                    PROV_R_PSS_SALTLEN_TOO_SMALL,
1177                                    "Should be more than %d, but would be "
1178                                    "set to match digest size (%d)",
1179                                    prsactx->min_saltlen,
1180                                    EVP_MD_size(prsactx->md));
1181                     return 0;
1182                 }
1183                 break;
1184             default:
1185                 if (saltlen >= 0 && saltlen < prsactx->min_saltlen) {
1186                     ERR_raise_data(ERR_LIB_PROV,
1187                                    PROV_R_PSS_SALTLEN_TOO_SMALL,
1188                                    "Should be more than %d, "
1189                                    "but would be set to %d",
1190                                    prsactx->min_saltlen, saltlen);
1191                     return 0;
1192                 }
1193             }
1194         }
1195
1196         prsactx->saltlen = saltlen;
1197     }
1198
1199     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_MGF1_DIGEST);
1200     if (p != NULL) {
1201         char mdname[OSSL_MAX_NAME_SIZE] = "", *pmdname = mdname;
1202         char mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmdprops = mdprops;
1203         const OSSL_PARAM *propsp =
1204             OSSL_PARAM_locate_const(params,
1205                                     OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES);
1206
1207         if (!OSSL_PARAM_get_utf8_string(p, &pmdname, sizeof(mdname)))
1208             return 0;
1209
1210         if (propsp == NULL)
1211             pmdprops = NULL;
1212         else if (!OSSL_PARAM_get_utf8_string(propsp,
1213                                              &pmdprops, sizeof(mdprops)))
1214             return 0;
1215
1216         if (prsactx->pad_mode != RSA_PKCS1_PSS_PADDING) {
1217             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MGF1_MD);
1218             return  0;
1219         }
1220
1221         if (rsa_pss_restricted(prsactx)) {
1222             /* TODO(3.0) figure out what to do for prsactx->mgf1_md == NULL */
1223             if (prsactx->mgf1_md == NULL
1224                 || EVP_MD_is_a(prsactx->mgf1_md, mdname))
1225                 return 1;
1226             ERR_raise(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED);
1227             return 0;
1228         }
1229
1230         /* non-PSS code follows */
1231         if (!rsa_setup_mgf1_md(prsactx, mdname, pmdprops))
1232             return 0;
1233     }
1234
1235     return 1;
1236 }
1237
1238 static const OSSL_PARAM known_settable_ctx_params[] = {
1239     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PAD_MODE, NULL, 0),
1240     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0),
1241     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PROPERTIES, NULL, 0),
1242     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_DIGEST, NULL, 0),
1243     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES, NULL, 0),
1244     OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PSS_SALTLEN, NULL, 0),
1245     OSSL_PARAM_END
1246 };
1247
1248 static const OSSL_PARAM *rsa_settable_ctx_params(ossl_unused void *provctx)
1249 {
1250     /*
1251      * TODO(3.0): Should this function return a different set of settable ctx
1252      * params if the ctx is being used for a DigestSign/DigestVerify? In that
1253      * case it is not allowed to set the digest size/digest name because the
1254      * digest is explicitly set as part of the init.
1255      */
1256     return known_settable_ctx_params;
1257 }
1258
1259 static int rsa_get_ctx_md_params(void *vprsactx, OSSL_PARAM *params)
1260 {
1261     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1262
1263     if (prsactx->mdctx == NULL)
1264         return 0;
1265
1266     return EVP_MD_CTX_get_params(prsactx->mdctx, params);
1267 }
1268
1269 static const OSSL_PARAM *rsa_gettable_ctx_md_params(void *vprsactx)
1270 {
1271     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1272
1273     if (prsactx->md == NULL)
1274         return 0;
1275
1276     return EVP_MD_gettable_ctx_params(prsactx->md);
1277 }
1278
1279 static int rsa_set_ctx_md_params(void *vprsactx, const OSSL_PARAM params[])
1280 {
1281     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1282
1283     if (prsactx->mdctx == NULL)
1284         return 0;
1285
1286     return EVP_MD_CTX_set_params(prsactx->mdctx, params);
1287 }
1288
1289 static const OSSL_PARAM *rsa_settable_ctx_md_params(void *vprsactx)
1290 {
1291     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
1292
1293     if (prsactx->md == NULL)
1294         return 0;
1295
1296     return EVP_MD_settable_ctx_params(prsactx->md);
1297 }
1298
1299 const OSSL_DISPATCH ossl_rsa_signature_functions[] = {
1300     { OSSL_FUNC_SIGNATURE_NEWCTX, (void (*)(void))rsa_newctx },
1301     { OSSL_FUNC_SIGNATURE_SIGN_INIT, (void (*)(void))rsa_sign_init },
1302     { OSSL_FUNC_SIGNATURE_SIGN, (void (*)(void))rsa_sign },
1303     { OSSL_FUNC_SIGNATURE_VERIFY_INIT, (void (*)(void))rsa_verify_init },
1304     { OSSL_FUNC_SIGNATURE_VERIFY, (void (*)(void))rsa_verify },
1305     { OSSL_FUNC_SIGNATURE_VERIFY_RECOVER_INIT,
1306       (void (*)(void))rsa_verify_recover_init },
1307     { OSSL_FUNC_SIGNATURE_VERIFY_RECOVER,
1308       (void (*)(void))rsa_verify_recover },
1309     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_INIT,
1310       (void (*)(void))rsa_digest_sign_init },
1311     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_UPDATE,
1312       (void (*)(void))rsa_digest_signverify_update },
1313     { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_FINAL,
1314       (void (*)(void))rsa_digest_sign_final },
1315     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_INIT,
1316       (void (*)(void))rsa_digest_verify_init },
1317     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_UPDATE,
1318       (void (*)(void))rsa_digest_signverify_update },
1319     { OSSL_FUNC_SIGNATURE_DIGEST_VERIFY_FINAL,
1320       (void (*)(void))rsa_digest_verify_final },
1321     { OSSL_FUNC_SIGNATURE_FREECTX, (void (*)(void))rsa_freectx },
1322     { OSSL_FUNC_SIGNATURE_DUPCTX, (void (*)(void))rsa_dupctx },
1323     { OSSL_FUNC_SIGNATURE_GET_CTX_PARAMS, (void (*)(void))rsa_get_ctx_params },
1324     { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_PARAMS,
1325       (void (*)(void))rsa_gettable_ctx_params },
1326     { OSSL_FUNC_SIGNATURE_SET_CTX_PARAMS, (void (*)(void))rsa_set_ctx_params },
1327     { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_PARAMS,
1328       (void (*)(void))rsa_settable_ctx_params },
1329     { OSSL_FUNC_SIGNATURE_GET_CTX_MD_PARAMS,
1330       (void (*)(void))rsa_get_ctx_md_params },
1331     { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_MD_PARAMS,
1332       (void (*)(void))rsa_gettable_ctx_md_params },
1333     { OSSL_FUNC_SIGNATURE_SET_CTX_MD_PARAMS,
1334       (void (*)(void))rsa_set_ctx_md_params },
1335     { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS,
1336       (void (*)(void))rsa_settable_ctx_md_params },
1337     { 0, NULL }
1338 };