crypto/rsa/rsa_ameth.c

   1 /*
   2  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
   3  * 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <stdio.h>
  60 #include "internal/cryptlib.h"
  61 #include <openssl/asn1t.h>
  62 #include <openssl/x509.h>
  63 #include <openssl/rsa.h>
  64 #include <openssl/bn.h>
  65 #include <openssl/cms.h>
  66 #include "internal/asn1_int.h"
  67 #include "internal/evp_int.h"
  68
  69 #ifndef OPENSSL_NO_CMS
  70 static int rsa_cms_sign(CMS_SignerInfo *si);
  71 static int rsa_cms_verify(CMS_SignerInfo *si);
  72 static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
  73 static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
  74 #endif
  75
  76 static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
  77 {
  78     unsigned char *penc = NULL;
  79     int penclen;
  80     penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
  81     if (penclen <= 0)
  82         return 0;
  83     if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_RSA),
  84                                V_ASN1_NULL, NULL, penc, penclen))
  85         return 1;
  86
  87     OPENSSL_free(penc);
  88     return 0;
  89 }
  90
  91 static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
  92 {
  93     const unsigned char *p;
  94     int pklen;
  95     RSA *rsa = NULL;
  96
  97     if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, NULL, pubkey))
  98         return 0;
  99     if ((rsa = d2i_RSAPublicKey(NULL, &p, pklen)) == NULL) {
 100         RSAerr(RSA_F_RSA_PUB_DECODE, ERR_R_RSA_LIB);
 101         return 0;
 102     }
 103     EVP_PKEY_assign_RSA(pkey, rsa);
 104     return 1;
 105 }
 106
 107 static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
 108 {
 109     if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0
 110         || BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
 111         return 0;
 112     return 1;
 113 }
 114
 115 static int old_rsa_priv_decode(EVP_PKEY *pkey,
 116                                const unsigned char **pder, int derlen)
 117 {
 118     RSA *rsa;
 119
 120     if ((rsa = d2i_RSAPrivateKey(NULL, pder, derlen)) == NULL) {
 121         RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB);
 122         return 0;
 123     }
 124     EVP_PKEY_assign_RSA(pkey, rsa);
 125     return 1;
 126 }
 127
 128 static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
 129 {
 130     return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
 131 }
 132
 133 static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
 134 {
 135     unsigned char *rk = NULL;
 136     int rklen;
 137     rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
 138
 139     if (rklen <= 0) {
 140         RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
 141         return 0;
 142     }
 143
 144     if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_rsaEncryption), 0,
 145                          V_ASN1_NULL, NULL, rk, rklen)) {
 146         RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
 147         return 0;
 148     }
 149
 150     return 1;
 151 }
 152
 153 static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8)
 154 {
 155     const unsigned char *p;
 156     int pklen;
 157     if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8))
 158         return 0;
 159     return old_rsa_priv_decode(pkey, &p, pklen);
 160 }
 161
 162 static int int_rsa_size(const EVP_PKEY *pkey)
 163 {
 164     return RSA_size(pkey->pkey.rsa);
 165 }
 166
 167 static int rsa_bits(const EVP_PKEY *pkey)
 168 {
 169     return BN_num_bits(pkey->pkey.rsa->n);
 170 }
 171
 172 static int rsa_security_bits(const EVP_PKEY *pkey)
 173 {
 174     return RSA_security_bits(pkey->pkey.rsa);
 175 }
 176
 177 static void int_rsa_free(EVP_PKEY *pkey)
 178 {
 179     RSA_free(pkey->pkey.rsa);
 180 }
 181
 182 static int do_rsa_print(BIO *bp, const RSA *x, int off, int priv)
 183 {
 184     char *str;
 185     const char *s;
 186     int ret = 0, mod_len = 0;
 187
 188     if (x->n != NULL)
 189         mod_len = BN_num_bits(x->n);
 190
 191     if (!BIO_indent(bp, off, 128))
 192         goto err;
 193
 194     if (priv && x->d) {
 195         if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len) <= 0)
 196             goto err;
 197         str = "modulus:";
 198         s = "publicExponent:";
 199     } else {
 200         if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len) <= 0)
 201             goto err;
 202         str = "Modulus:";
 203         s = "Exponent:";
 204     }
 205     if (!ASN1_bn_print(bp, str, x->n, NULL, off))
 206         goto err;
 207     if (!ASN1_bn_print(bp, s, x->e, NULL, off))
 208         goto err;
 209     if (priv) {
 210         if (!ASN1_bn_print(bp, "privateExponent:", x->d, NULL, off))
 211             goto err;
 212         if (!ASN1_bn_print(bp, "prime1:", x->p, NULL, off))
 213             goto err;
 214         if (!ASN1_bn_print(bp, "prime2:", x->q, NULL, off))
 215             goto err;
 216         if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, NULL, off))
 217             goto err;
 218         if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, NULL, off))
 219             goto err;
 220         if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, NULL, off))
 221             goto err;
 222     }
 223     ret = 1;
 224  err:
 225     return (ret);
 226 }
 227
 228 static int rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent,
 229                          ASN1_PCTX *ctx)
 230 {
 231     return do_rsa_print(bp, pkey->pkey.rsa, indent, 0);
 232 }
 233
 234 static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
 235                           ASN1_PCTX *ctx)
 236 {
 237     return do_rsa_print(bp, pkey->pkey.rsa, indent, 1);
 238 }
 239
 240 /* Given an MGF1 Algorithm ID decode to an Algorithm Identifier */
 241 static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg)
 242 {
 243     if (alg == NULL)
 244         return NULL;
 245     if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
 246         return NULL;
 247     return ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(X509_ALGOR),
 248                                      alg->parameter);
 249 }
 250
 251 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg,
 252                                       X509_ALGOR **pmaskHash)
 253 {
 254     RSA_PSS_PARAMS *pss;
 255
 256     *pmaskHash = NULL;
 257
 258     pss = ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(RSA_PSS_PARAMS),
 259                                     alg->parameter);
 260
 261     if (!pss)
 262         return NULL;
 263
 264     *pmaskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
 265
 266     return pss;
 267 }
 268
 269 static int rsa_pss_param_print(BIO *bp, RSA_PSS_PARAMS *pss,
 270                                X509_ALGOR *maskHash, int indent)
 271 {
 272     int rv = 0;
 273     if (!pss) {
 274         if (BIO_puts(bp, " (INVALID PSS PARAMETERS)\n") <= 0)
 275             return 0;
 276         return 1;
 277     }
 278     if (BIO_puts(bp, "\n") <= 0)
 279         goto err;
 280     if (!BIO_indent(bp, indent, 128))
 281         goto err;
 282     if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
 283         goto err;
 284
 285     if (pss->hashAlgorithm) {
 286         if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
 287             goto err;
 288     } else if (BIO_puts(bp, "sha1 (default)") <= 0)
 289         goto err;
 290
 291     if (BIO_puts(bp, "\n") <= 0)
 292         goto err;
 293
 294     if (!BIO_indent(bp, indent, 128))
 295         goto err;
 296
 297     if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
 298         goto err;
 299     if (pss->maskGenAlgorithm) {
 300         if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
 301             goto err;
 302         if (BIO_puts(bp, " with ") <= 0)
 303             goto err;
 304         if (maskHash) {
 305             if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
 306                 goto err;
 307         } else if (BIO_puts(bp, "INVALID") <= 0)
 308             goto err;
 309     } else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0)
 310         goto err;
 311     BIO_puts(bp, "\n");
 312
 313     if (!BIO_indent(bp, indent, 128))
 314         goto err;
 315     if (BIO_puts(bp, "Salt Length: 0x") <= 0)
 316         goto err;
 317     if (pss->saltLength) {
 318         if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
 319             goto err;
 320     } else if (BIO_puts(bp, "14 (default)") <= 0)
 321         goto err;
 322     BIO_puts(bp, "\n");
 323
 324     if (!BIO_indent(bp, indent, 128))
 325         goto err;
 326     if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
 327         goto err;
 328     if (pss->trailerField) {
 329         if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
 330             goto err;
 331     } else if (BIO_puts(bp, "BC (default)") <= 0)
 332         goto err;
 333     BIO_puts(bp, "\n");
 334
 335     rv = 1;
 336
 337  err:
 338     return rv;
 339
 340 }
 341
 342 static int rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
 343                          const ASN1_STRING *sig, int indent, ASN1_PCTX *pctx)
 344 {
 345     if (OBJ_obj2nid(sigalg->algorithm) == NID_rsassaPss) {
 346         int rv;
 347         RSA_PSS_PARAMS *pss;
 348         X509_ALGOR *maskHash;
 349         pss = rsa_pss_decode(sigalg, &maskHash);
 350         rv = rsa_pss_param_print(bp, pss, maskHash, indent);
 351         RSA_PSS_PARAMS_free(pss);
 352         X509_ALGOR_free(maskHash);
 353         if (!rv)
 354             return 0;
 355     } else if (!sig && BIO_puts(bp, "\n") <= 0)
 356         return 0;
 357     if (sig)
 358         return X509_signature_dump(bp, sig, indent);
 359     return 1;
 360 }
 361
 362 static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 363 {
 364     X509_ALGOR *alg = NULL;
 365     switch (op) {
 366
 367     case ASN1_PKEY_CTRL_PKCS7_SIGN:
 368         if (arg1 == 0)
 369             PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
 370         break;
 371
 372     case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
 373         if (arg1 == 0)
 374             PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
 375         break;
 376 #ifndef OPENSSL_NO_CMS
 377     case ASN1_PKEY_CTRL_CMS_SIGN:
 378         if (arg1 == 0)
 379             return rsa_cms_sign(arg2);
 380         else if (arg1 == 1)
 381             return rsa_cms_verify(arg2);
 382         break;
 383
 384     case ASN1_PKEY_CTRL_CMS_ENVELOPE:
 385         if (arg1 == 0)
 386             return rsa_cms_encrypt(arg2);
 387         else if (arg1 == 1)
 388             return rsa_cms_decrypt(arg2);
 389         break;
 390
 391     case ASN1_PKEY_CTRL_CMS_RI_TYPE:
 392         *(int *)arg2 = CMS_RECIPINFO_TRANS;
 393         return 1;
 394 #endif
 395
 396     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
 397         *(int *)arg2 = NID_sha256;
 398         return 1;
 399
 400     default:
 401         return -2;
 402
 403     }
 404
 405     if (alg)
 406         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
 407
 408     return 1;
 409
 410 }
 411
 412 /* allocate and set algorithm ID from EVP_MD, default SHA1 */
 413 static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
 414 {
 415     if (EVP_MD_type(md) == NID_sha1)
 416         return 1;
 417     *palg = X509_ALGOR_new();
 418     if (*palg == NULL)
 419         return 0;
 420     X509_ALGOR_set_md(*palg, md);
 421     return 1;
 422 }
 423
 424 /* Allocate and set MGF1 algorithm ID from EVP_MD */
 425 static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
 426 {
 427     X509_ALGOR *algtmp = NULL;
 428     ASN1_STRING *stmp = NULL;
 429     *palg = NULL;
 430     if (EVP_MD_type(mgf1md) == NID_sha1)
 431         return 1;
 432     /* need to embed algorithm ID inside another */
 433     if (!rsa_md_to_algor(&algtmp, mgf1md))
 434         goto err;
 435     if (!ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp))
 436          goto err;
 437     *palg = X509_ALGOR_new();
 438     if (*palg == NULL)
 439         goto err;
 440     X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
 441     stmp = NULL;
 442  err:
 443     ASN1_STRING_free(stmp);
 444     X509_ALGOR_free(algtmp);
 445     if (*palg)
 446         return 1;
 447     return 0;
 448 }
 449
 450 /* convert algorithm ID to EVP_MD, default SHA1 */
 451 static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg)
 452 {
 453     const EVP_MD *md;
 454     if (!alg)
 455         return EVP_sha1();
 456     md = EVP_get_digestbyobj(alg->algorithm);
 457     if (md == NULL)
 458         RSAerr(RSA_F_RSA_ALGOR_TO_MD, RSA_R_UNKNOWN_DIGEST);
 459     return md;
 460 }
 461
 462 /* convert MGF1 algorithm ID to EVP_MD, default SHA1 */
 463 static const EVP_MD *rsa_mgf1_to_md(X509_ALGOR *alg, X509_ALGOR *maskHash)
 464 {
 465     const EVP_MD *md;
 466     if (!alg)
 467         return EVP_sha1();
 468     /* Check mask and lookup mask hash algorithm */
 469     if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) {
 470         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_ALGORITHM);
 471         return NULL;
 472     }
 473     if (!maskHash) {
 474         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_PARAMETER);
 475         return NULL;
 476     }
 477     md = EVP_get_digestbyobj(maskHash->algorithm);
 478     if (md == NULL) {
 479         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNKNOWN_MASK_DIGEST);
 480         return NULL;
 481     }
 482     return md;
 483 }
 484
 485 /*
 486  * Convert EVP_PKEY_CTX is PSS mode into corresponding algorithm parameter,
 487  * suitable for setting an AlgorithmIdentifier.
 488  */
 489
 490 static ASN1_STRING *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
 491 {
 492     const EVP_MD *sigmd, *mgf1md;
 493     RSA_PSS_PARAMS *pss = NULL;
 494     ASN1_STRING *os = NULL;
 495     EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
 496     int saltlen, rv = 0;
 497     if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
 498         goto err;
 499     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
 500         goto err;
 501     if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
 502         goto err;
 503     if (saltlen == -1)
 504         saltlen = EVP_MD_size(sigmd);
 505     else if (saltlen == -2) {
 506         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
 507         if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
 508             saltlen--;
 509     }
 510     pss = RSA_PSS_PARAMS_new();
 511     if (pss == NULL)
 512         goto err;
 513     if (saltlen != 20) {
 514         pss->saltLength = ASN1_INTEGER_new();
 515         if (pss->saltLength == NULL)
 516             goto err;
 517         if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
 518             goto err;
 519     }
 520     if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
 521         goto err;
 522     if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
 523         goto err;
 524     /* Finally create string with pss parameter encoding. */
 525     if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os))
 526          goto err;
 527     rv = 1;
 528  err:
 529     RSA_PSS_PARAMS_free(pss);
 530     if (rv)
 531         return os;
 532     ASN1_STRING_free(os);
 533     return NULL;
 534 }
 535
 536 /*
 537  * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
 538  * then the EVP_MD_CTX is setup and initialised. If it is NULL parameters are
 539  * passed to pkctx instead.
 540  */
 541
 542 static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
 543                           X509_ALGOR *sigalg, EVP_PKEY *pkey)
 544 {
 545     int rv = -1;
 546     int saltlen;
 547     const EVP_MD *mgf1md = NULL, *md = NULL;
 548     RSA_PSS_PARAMS *pss;
 549     X509_ALGOR *maskHash;
 550     /* Sanity check: make sure it is PSS */
 551     if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
 552         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
 553         return -1;
 554     }
 555     /* Decode PSS parameters */
 556     pss = rsa_pss_decode(sigalg, &maskHash);
 557
 558     if (pss == NULL) {
 559         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_PSS_PARAMETERS);
 560         goto err;
 561     }
 562     mgf1md = rsa_mgf1_to_md(pss->maskGenAlgorithm, maskHash);
 563     if (!mgf1md)
 564         goto err;
 565     md = rsa_algor_to_md(pss->hashAlgorithm);
 566     if (!md)
 567         goto err;
 568
 569     if (pss->saltLength) {
 570         saltlen = ASN1_INTEGER_get(pss->saltLength);
 571
 572         /*
 573          * Could perform more salt length sanity checks but the main RSA
 574          * routines will trap other invalid values anyway.
 575          */
 576         if (saltlen < 0) {
 577             RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_SALT_LENGTH);
 578             goto err;
 579         }
 580     } else
 581         saltlen = 20;
 582
 583     /*
 584      * low-level routines support only trailer field 0xbc (value 1) and
 585      * PKCS#1 says we should reject any other value anyway.
 586      */
 587     if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
 588         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_TRAILER);
 589         goto err;
 590     }
 591
 592     /* We have all parameters now set up context */
 593
 594     if (pkey) {
 595         if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
 596             goto err;
 597     } else {
 598         const EVP_MD *checkmd;
 599         if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
 600             goto err;
 601         if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
 602             RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_DIGEST_DOES_NOT_MATCH);
 603             goto err;
 604         }
 605     }
 606
 607     if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
 608         goto err;
 609
 610     if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
 611         goto err;
 612
 613     if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
 614         goto err;
 615     /* Carry on */
 616     rv = 1;
 617
 618  err:
 619     RSA_PSS_PARAMS_free(pss);
 620     X509_ALGOR_free(maskHash);
 621     return rv;
 622 }
 623
 624 #ifndef OPENSSL_NO_CMS
 625 static int rsa_cms_verify(CMS_SignerInfo *si)
 626 {
 627     int nid, nid2;
 628     X509_ALGOR *alg;
 629     EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
 630     CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
 631     nid = OBJ_obj2nid(alg->algorithm);
 632     if (nid == NID_rsaEncryption)
 633         return 1;
 634     if (nid == NID_rsassaPss)
 635         return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
 636     /* Workaround for some implementation that use a signature OID */
 637     if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
 638         if (nid2 == NID_rsaEncryption)
 639             return 1;
 640     }
 641     return 0;
 642 }
 643 #endif
 644
 645 /*
 646  * Customised RSA item verification routine. This is called when a signature
 647  * is encountered requiring special handling. We currently only handle PSS.
 648  */
 649
 650 static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
 651                            X509_ALGOR *sigalg, ASN1_BIT_STRING *sig,
 652                            EVP_PKEY *pkey)
 653 {
 654     /* Sanity check: make sure it is PSS */
 655     if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
 656         RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
 657         return -1;
 658     }
 659     if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
 660         /* Carry on */
 661         return 2;
 662     }
 663     return -1;
 664 }
 665
 666 #ifndef OPENSSL_NO_CMS
 667 static int rsa_cms_sign(CMS_SignerInfo *si)
 668 {
 669     int pad_mode = RSA_PKCS1_PADDING;
 670     X509_ALGOR *alg;
 671     EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
 672     ASN1_STRING *os = NULL;
 673     CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
 674     if (pkctx) {
 675         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
 676             return 0;
 677     }
 678     if (pad_mode == RSA_PKCS1_PADDING) {
 679         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
 680         return 1;
 681     }
 682     /* We don't support it */
 683     if (pad_mode != RSA_PKCS1_PSS_PADDING)
 684         return 0;
 685     os = rsa_ctx_to_pss(pkctx);
 686     if (!os)
 687         return 0;
 688     X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os);
 689     return 1;
 690 }
 691 #endif
 692
 693 static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
 694                          X509_ALGOR *alg1, X509_ALGOR *alg2,
 695                          ASN1_BIT_STRING *sig)
 696 {
 697     int pad_mode;
 698     EVP_PKEY_CTX *pkctx = EVP_MD_CTX_pkey_ctx(ctx);
 699     if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
 700         return 0;
 701     if (pad_mode == RSA_PKCS1_PADDING)
 702         return 2;
 703     if (pad_mode == RSA_PKCS1_PSS_PADDING) {
 704         ASN1_STRING *os1 = NULL;
 705         os1 = rsa_ctx_to_pss(pkctx);
 706         if (!os1)
 707             return 0;
 708         /* Duplicate parameters if we have to */
 709         if (alg2) {
 710             ASN1_STRING *os2 = ASN1_STRING_dup(os1);
 711             if (!os2) {
 712                 ASN1_STRING_free(os1);
 713                 return 0;
 714             }
 715             X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_rsassaPss),
 716                             V_ASN1_SEQUENCE, os2);
 717         }
 718         X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_rsassaPss),
 719                         V_ASN1_SEQUENCE, os1);
 720         return 3;
 721     }
 722     return 2;
 723 }
 724
 725 #ifndef OPENSSL_NO_CMS
 726 static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg,
 727                                         X509_ALGOR **pmaskHash)
 728 {
 729     RSA_OAEP_PARAMS *pss;
 730
 731     *pmaskHash = NULL;
 732
 733     pss = ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(RSA_OAEP_PARAMS),
 734                                     alg->parameter);
 735
 736     if (!pss)
 737         return NULL;
 738
 739     *pmaskHash = rsa_mgf1_decode(pss->maskGenFunc);
 740
 741     return pss;
 742 }
 743
 744 static int rsa_cms_decrypt(CMS_RecipientInfo *ri)
 745 {
 746     EVP_PKEY_CTX *pkctx;
 747     X509_ALGOR *cmsalg;
 748     int nid;
 749     int rv = -1;
 750     unsigned char *label = NULL;
 751     int labellen = 0;
 752     const EVP_MD *mgf1md = NULL, *md = NULL;
 753     RSA_OAEP_PARAMS *oaep;
 754     X509_ALGOR *maskHash;
 755     pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
 756     if (!pkctx)
 757         return 0;
 758     if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
 759         return -1;
 760     nid = OBJ_obj2nid(cmsalg->algorithm);
 761     if (nid == NID_rsaEncryption)
 762         return 1;
 763     if (nid != NID_rsaesOaep) {
 764         RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
 765         return -1;
 766     }
 767     /* Decode OAEP parameters */
 768     oaep = rsa_oaep_decode(cmsalg, &maskHash);
 769
 770     if (oaep == NULL) {
 771         RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_OAEP_PARAMETERS);
 772         goto err;
 773     }
 774
 775     mgf1md = rsa_mgf1_to_md(oaep->maskGenFunc, maskHash);
 776     if (!mgf1md)
 777         goto err;
 778     md = rsa_algor_to_md(oaep->hashFunc);
 779     if (!md)
 780         goto err;
 781
 782     if (oaep->pSourceFunc) {
 783         X509_ALGOR *plab = oaep->pSourceFunc;
 784         if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
 785             RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_LABEL_SOURCE);
 786             goto err;
 787         }
 788         if (plab->parameter->type != V_ASN1_OCTET_STRING) {
 789             RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_LABEL);
 790             goto err;
 791         }
 792
 793         label = plab->parameter->value.octet_string->data;
 794         /* Stop label being freed when OAEP parameters are freed */
 795         plab->parameter->value.octet_string->data = NULL;
 796         labellen = plab->parameter->value.octet_string->length;
 797     }
 798
 799     if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
 800         goto err;
 801     if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
 802         goto err;
 803     if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
 804         goto err;
 805     if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
 806         goto err;
 807     /* Carry on */
 808     rv = 1;
 809
 810  err:
 811     RSA_OAEP_PARAMS_free(oaep);
 812     X509_ALGOR_free(maskHash);
 813     return rv;
 814 }
 815
 816 static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
 817 {
 818     const EVP_MD *md, *mgf1md;
 819     RSA_OAEP_PARAMS *oaep = NULL;
 820     ASN1_STRING *os = NULL;
 821     X509_ALGOR *alg;
 822     EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
 823     int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
 824     unsigned char *label;
 825     CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg);
 826     if (pkctx) {
 827         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
 828             return 0;
 829     }
 830     if (pad_mode == RSA_PKCS1_PADDING) {
 831         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
 832         return 1;
 833     }
 834     /* Not supported */
 835     if (pad_mode != RSA_PKCS1_OAEP_PADDING)
 836         return 0;
 837     if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
 838         goto err;
 839     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
 840         goto err;
 841     labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
 842     if (labellen < 0)
 843         goto err;
 844     oaep = RSA_OAEP_PARAMS_new();
 845     if (oaep == NULL)
 846         goto err;
 847     if (!rsa_md_to_algor(&oaep->hashFunc, md))
 848         goto err;
 849     if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
 850         goto err;
 851     if (labellen > 0) {
 852         ASN1_OCTET_STRING *los = ASN1_OCTET_STRING_new();
 853         oaep->pSourceFunc = X509_ALGOR_new();
 854         if (oaep->pSourceFunc == NULL)
 855             goto err;
 856         if (los == NULL)
 857             goto err;
 858         if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
 859             ASN1_OCTET_STRING_free(los);
 860             goto err;
 861         }
 862         X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
 863                         V_ASN1_OCTET_STRING, los);
 864     }
 865     /* create string with pss parameter encoding. */
 866     if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os))
 867          goto err;
 868     X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
 869     os = NULL;
 870     rv = 1;
 871  err:
 872     RSA_OAEP_PARAMS_free(oaep);
 873     ASN1_STRING_free(os);
 874     return rv;
 875 }
 876 #endif
 877
 878 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
 879     {
 880      EVP_PKEY_RSA,
 881      EVP_PKEY_RSA,
 882      ASN1_PKEY_SIGPARAM_NULL,
 883
 884      "RSA",
 885      "OpenSSL RSA method",
 886
 887      rsa_pub_decode,
 888      rsa_pub_encode,
 889      rsa_pub_cmp,
 890      rsa_pub_print,
 891
 892      rsa_priv_decode,
 893      rsa_priv_encode,
 894      rsa_priv_print,
 895
 896      int_rsa_size,
 897      rsa_bits,
 898      rsa_security_bits,
 899
 900      0, 0, 0, 0, 0, 0,
 901
 902      rsa_sig_print,
 903      int_rsa_free,
 904      rsa_pkey_ctrl,
 905      old_rsa_priv_decode,
 906      old_rsa_priv_encode,
 907      rsa_item_verify,
 908      rsa_item_sign},
 909
 910     {
 911      EVP_PKEY_RSA2,
 912      EVP_PKEY_RSA,
 913      ASN1_PKEY_ALIAS}
 914 };