crypto/rsa/rsa_pmeth.c

   1 /*
   2  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
   3  * 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <stdio.h>
  60 #include "internal/cryptlib.h"
  61 #include <openssl/asn1t.h>
  62 #include <openssl/x509.h>
  63 #include <openssl/rsa.h>
  64 #include <openssl/bn.h>
  65 #include <openssl/evp.h>
  66 #include <openssl/x509v3.h>
  67 #ifndef OPENSSL_NO_CMS
  68 # include <openssl/cms.h>
  69 #endif
  70 #include "internal/evp_int.h"
  71 #include "rsa_locl.h"
  72
  73 /* RSA pkey context structure */
  74
  75 typedef struct {
  76     /* Key gen parameters */
  77     int nbits;
  78     BIGNUM *pub_exp;
  79     /* Keygen callback info */
  80     int gentmp[2];
  81     /* RSA padding mode */
  82     int pad_mode;
  83     /* message digest */
  84     const EVP_MD *md;
  85     /* message digest for MGF1 */
  86     const EVP_MD *mgf1md;
  87     /* PSS salt length */
  88     int saltlen;
  89     /* Temp buffer */
  90     unsigned char *tbuf;
  91     /* OAEP label */
  92     unsigned char *oaep_label;
  93     size_t oaep_labellen;
  94 } RSA_PKEY_CTX;
  95
  96 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
  97 {
  98     RSA_PKEY_CTX *rctx;
  99     rctx = OPENSSL_zalloc(sizeof(*rctx));
 100     if (rctx == NULL)
 101         return 0;
 102     rctx->nbits = 1024;
 103     rctx->pad_mode = RSA_PKCS1_PADDING;
 104     rctx->saltlen = -2;
 105     ctx->data = rctx;
 106     ctx->keygen_info = rctx->gentmp;
 107     ctx->keygen_info_count = 2;
 108
 109     return 1;
 110 }
 111
 112 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 113 {
 114     RSA_PKEY_CTX *dctx, *sctx;
 115     if (!pkey_rsa_init(dst))
 116         return 0;
 117     sctx = src->data;
 118     dctx = dst->data;
 119     dctx->nbits = sctx->nbits;
 120     if (sctx->pub_exp) {
 121         dctx->pub_exp = BN_dup(sctx->pub_exp);
 122         if (!dctx->pub_exp)
 123             return 0;
 124     }
 125     dctx->pad_mode = sctx->pad_mode;
 126     dctx->md = sctx->md;
 127     dctx->mgf1md = sctx->mgf1md;
 128     if (sctx->oaep_label) {
 129         OPENSSL_free(dctx->oaep_label);
 130         dctx->oaep_label = OPENSSL_memdup(sctx->oaep_label, sctx->oaep_labellen);
 131         if (!dctx->oaep_label)
 132             return 0;
 133         dctx->oaep_labellen = sctx->oaep_labellen;
 134     }
 135     return 1;
 136 }
 137
 138 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 139 {
 140     if (ctx->tbuf)
 141         return 1;
 142     ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 143     if (ctx->tbuf == NULL)
 144         return 0;
 145     return 1;
 146 }
 147
 148 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 149 {
 150     RSA_PKEY_CTX *rctx = ctx->data;
 151     if (rctx) {
 152         BN_free(rctx->pub_exp);
 153         OPENSSL_free(rctx->tbuf);
 154         OPENSSL_free(rctx->oaep_label);
 155         OPENSSL_free(rctx);
 156     }
 157 }
 158
 159 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
 160                          size_t *siglen, const unsigned char *tbs,
 161                          size_t tbslen)
 162 {
 163     int ret;
 164     RSA_PKEY_CTX *rctx = ctx->data;
 165     RSA *rsa = ctx->pkey->pkey.rsa;
 166
 167     if (rctx->md) {
 168         if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
 169             RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_INVALID_DIGEST_LENGTH);
 170             return -1;
 171         }
 172
 173         if (EVP_MD_type(rctx->md) == NID_mdc2) {
 174             unsigned int sltmp;
 175             if (rctx->pad_mode != RSA_PKCS1_PADDING)
 176                 return -1;
 177             ret = RSA_sign_ASN1_OCTET_STRING(0,
 178                                              tbs, tbslen, sig, &sltmp, rsa);
 179
 180             if (ret <= 0)
 181                 return ret;
 182             ret = sltmp;
 183         } else if (rctx->pad_mode == RSA_X931_PADDING) {
 184             if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
 185                 RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_KEY_SIZE_TOO_SMALL);
 186                 return -1;
 187             }
 188             if (!setup_tbuf(rctx, ctx)) {
 189                 RSAerr(RSA_F_PKEY_RSA_SIGN, ERR_R_MALLOC_FAILURE);
 190                 return -1;
 191             }
 192             memcpy(rctx->tbuf, tbs, tbslen);
 193             rctx->tbuf[tbslen] = RSA_X931_hash_id(EVP_MD_type(rctx->md));
 194             ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 195                                       sig, rsa, RSA_X931_PADDING);
 196         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
 197             unsigned int sltmp;
 198             ret = RSA_sign(EVP_MD_type(rctx->md),
 199                            tbs, tbslen, sig, &sltmp, rsa);
 200             if (ret <= 0)
 201                 return ret;
 202             ret = sltmp;
 203         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 204             if (!setup_tbuf(rctx, ctx))
 205                 return -1;
 206             if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
 207                                                 rctx->tbuf, tbs,
 208                                                 rctx->md, rctx->mgf1md,
 209                                                 rctx->saltlen))
 210                 return -1;
 211             ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 212                                       sig, rsa, RSA_NO_PADDING);
 213         } else
 214             return -1;
 215     } else
 216         ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 217                                   rctx->pad_mode);
 218     if (ret < 0)
 219         return ret;
 220     *siglen = ret;
 221     return 1;
 222 }
 223
 224 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 225                                   unsigned char *rout, size_t *routlen,
 226                                   const unsigned char *sig, size_t siglen)
 227 {
 228     int ret;
 229     RSA_PKEY_CTX *rctx = ctx->data;
 230
 231     if (rctx->md) {
 232         if (rctx->pad_mode == RSA_X931_PADDING) {
 233             if (!setup_tbuf(rctx, ctx))
 234                 return -1;
 235             ret = RSA_public_decrypt(siglen, sig,
 236                                      rctx->tbuf, ctx->pkey->pkey.rsa,
 237                                      RSA_X931_PADDING);
 238             if (ret < 1)
 239                 return 0;
 240             ret--;
 241             if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
 242                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 243                        RSA_R_ALGORITHM_MISMATCH);
 244                 return 0;
 245             }
 246             if (ret != EVP_MD_size(rctx->md)) {
 247                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 248                        RSA_R_INVALID_DIGEST_LENGTH);
 249                 return 0;
 250             }
 251             if (rout)
 252                 memcpy(rout, rctx->tbuf, ret);
 253         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
 254             size_t sltmp;
 255             ret = int_rsa_verify(EVP_MD_type(rctx->md),
 256                                  NULL, 0, rout, &sltmp,
 257                                  sig, siglen, ctx->pkey->pkey.rsa);
 258             if (ret <= 0)
 259                 return 0;
 260             ret = sltmp;
 261         } else
 262             return -1;
 263     } else
 264         ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 265                                  rctx->pad_mode);
 266     if (ret < 0)
 267         return ret;
 268     *routlen = ret;
 269     return 1;
 270 }
 271
 272 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 273                            const unsigned char *sig, size_t siglen,
 274                            const unsigned char *tbs, size_t tbslen)
 275 {
 276     RSA_PKEY_CTX *rctx = ctx->data;
 277     RSA *rsa = ctx->pkey->pkey.rsa;
 278     size_t rslen;
 279     if (rctx->md) {
 280         if (rctx->pad_mode == RSA_PKCS1_PADDING)
 281             return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 282                               sig, siglen, rsa);
 283         if (rctx->pad_mode == RSA_X931_PADDING) {
 284             if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig, siglen) <= 0)
 285                 return 0;
 286         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 287             int ret;
 288             if (!setup_tbuf(rctx, ctx))
 289                 return -1;
 290             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 291                                      rsa, RSA_NO_PADDING);
 292             if (ret <= 0)
 293                 return 0;
 294             ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
 295                                             rctx->md, rctx->mgf1md,
 296                                             rctx->tbuf, rctx->saltlen);
 297             if (ret <= 0)
 298                 return 0;
 299             return 1;
 300         } else
 301             return -1;
 302     } else {
 303         if (!setup_tbuf(rctx, ctx))
 304             return -1;
 305         rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 306                                    rsa, rctx->pad_mode);
 307         if (rslen == 0)
 308             return 0;
 309     }
 310
 311     if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 312         return 0;
 313
 314     return 1;
 315
 316 }
 317
 318 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
 319                             unsigned char *out, size_t *outlen,
 320                             const unsigned char *in, size_t inlen)
 321 {
 322     int ret;
 323     RSA_PKEY_CTX *rctx = ctx->data;
 324     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
 325         int klen = RSA_size(ctx->pkey->pkey.rsa);
 326         if (!setup_tbuf(rctx, ctx))
 327             return -1;
 328         if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
 329                                              in, inlen,
 330                                              rctx->oaep_label,
 331                                              rctx->oaep_labellen,
 332                                              rctx->md, rctx->mgf1md))
 333             return -1;
 334         ret = RSA_public_encrypt(klen, rctx->tbuf, out,
 335                                  ctx->pkey->pkey.rsa, RSA_NO_PADDING);
 336     } else
 337         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 338                                  rctx->pad_mode);
 339     if (ret < 0)
 340         return ret;
 341     *outlen = ret;
 342     return 1;
 343 }
 344
 345 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
 346                             unsigned char *out, size_t *outlen,
 347                             const unsigned char *in, size_t inlen)
 348 {
 349     int ret;
 350     RSA_PKEY_CTX *rctx = ctx->data;
 351     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
 352         int i;
 353         if (!setup_tbuf(rctx, ctx))
 354             return -1;
 355         ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
 356                                   ctx->pkey->pkey.rsa, RSA_NO_PADDING);
 357         if (ret <= 0)
 358             return ret;
 359         for (i = 0; i < ret; i++) {
 360             if (rctx->tbuf[i])
 361                 break;
 362         }
 363         ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf + i,
 364                                                 ret - i, ret,
 365                                                 rctx->oaep_label,
 366                                                 rctx->oaep_labellen,
 367                                                 rctx->md, rctx->mgf1md);
 368     } else
 369         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 370                                   rctx->pad_mode);
 371     if (ret < 0)
 372         return ret;
 373     *outlen = ret;
 374     return 1;
 375 }
 376
 377 static int check_padding_md(const EVP_MD *md, int padding)
 378 {
 379     int mdnid;
 380     if (!md)
 381         return 1;
 382
 383     mdnid = EVP_MD_type(md);
 384
 385     if (padding == RSA_NO_PADDING) {
 386         RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
 387         return 0;
 388     }
 389
 390     if (padding == RSA_X931_PADDING) {
 391         if (RSA_X931_hash_id(mdnid) == -1) {
 392             RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_X931_DIGEST);
 393             return 0;
 394         }
 395     } else {
 396         switch(mdnid) {
 397         /* List of all supported RSA digests */
 398         case NID_sha1:
 399         case NID_sha224:
 400         case NID_sha256:
 401         case NID_sha384:
 402         case NID_sha512:
 403         case NID_md5:
 404         case NID_md5_sha1:
 405         case NID_md2:
 406         case NID_md4:
 407         case NID_mdc2:
 408         case NID_ripemd160:
 409             return 1;
 410
 411         default:
 412             RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_DIGEST);
 413             return 0;
 414
 415         }
 416     }
 417
 418     return 1;
 419 }
 420
 421 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 422 {
 423     RSA_PKEY_CTX *rctx = ctx->data;
 424     switch (type) {
 425     case EVP_PKEY_CTRL_RSA_PADDING:
 426         if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING)) {
 427             if (!check_padding_md(rctx->md, p1))
 428                 return 0;
 429             if (p1 == RSA_PKCS1_PSS_PADDING) {
 430                 if (!(ctx->operation &
 431                       (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
 432                     goto bad_pad;
 433                 if (!rctx->md)
 434                     rctx->md = EVP_sha1();
 435             }
 436             if (p1 == RSA_PKCS1_OAEP_PADDING) {
 437                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 438                     goto bad_pad;
 439                 if (!rctx->md)
 440                     rctx->md = EVP_sha1();
 441             }
 442             rctx->pad_mode = p1;
 443             return 1;
 444         }
 445  bad_pad:
 446         RSAerr(RSA_F_PKEY_RSA_CTRL,
 447                RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
 448         return -2;
 449
 450     case EVP_PKEY_CTRL_GET_RSA_PADDING:
 451         *(int *)p2 = rctx->pad_mode;
 452         return 1;
 453
 454     case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 455     case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
 456         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
 457             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
 458             return -2;
 459         }
 460         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
 461             *(int *)p2 = rctx->saltlen;
 462         else {
 463             if (p1 < -2)
 464                 return -2;
 465             rctx->saltlen = p1;
 466         }
 467         return 1;
 468
 469     case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 470         if (p1 < 512) {
 471             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_KEY_SIZE_TOO_SMALL);
 472             return -2;
 473         }
 474         rctx->nbits = p1;
 475         return 1;
 476
 477     case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 478         if (!p2)
 479             return -2;
 480         BN_free(rctx->pub_exp);
 481         rctx->pub_exp = p2;
 482         return 1;
 483
 484     case EVP_PKEY_CTRL_RSA_OAEP_MD:
 485     case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
 486         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
 487             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 488             return -2;
 489         }
 490         if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
 491             *(const EVP_MD **)p2 = rctx->md;
 492         else
 493             rctx->md = p2;
 494         return 1;
 495
 496     case EVP_PKEY_CTRL_MD:
 497         if (!check_padding_md(p2, rctx->pad_mode))
 498             return 0;
 499         rctx->md = p2;
 500         return 1;
 501
 502     case EVP_PKEY_CTRL_GET_MD:
 503         *(const EVP_MD **)p2 = rctx->md;
 504         return 1;
 505
 506     case EVP_PKEY_CTRL_RSA_MGF1_MD:
 507     case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
 508         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
 509             && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
 510             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
 511             return -2;
 512         }
 513         if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
 514             if (rctx->mgf1md)
 515                 *(const EVP_MD **)p2 = rctx->mgf1md;
 516             else
 517                 *(const EVP_MD **)p2 = rctx->md;
 518         } else
 519             rctx->mgf1md = p2;
 520         return 1;
 521
 522     case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
 523         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
 524             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 525             return -2;
 526         }
 527         OPENSSL_free(rctx->oaep_label);
 528         if (p2 && p1 > 0) {
 529             rctx->oaep_label = p2;
 530             rctx->oaep_labellen = p1;
 531         } else {
 532             rctx->oaep_label = NULL;
 533             rctx->oaep_labellen = 0;
 534         }
 535         return 1;
 536
 537     case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
 538         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
 539             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 540             return -2;
 541         }
 542         *(unsigned char **)p2 = rctx->oaep_label;
 543         return rctx->oaep_labellen;
 544
 545     case EVP_PKEY_CTRL_DIGESTINIT:
 546     case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 547     case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 548     case EVP_PKEY_CTRL_PKCS7_SIGN:
 549         return 1;
 550 #ifndef OPENSSL_NO_CMS
 551     case EVP_PKEY_CTRL_CMS_DECRYPT:
 552     case EVP_PKEY_CTRL_CMS_ENCRYPT:
 553     case EVP_PKEY_CTRL_CMS_SIGN:
 554         return 1;
 555 #endif
 556     case EVP_PKEY_CTRL_PEER_KEY:
 557         RSAerr(RSA_F_PKEY_RSA_CTRL,
 558                RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
 559         return -2;
 560
 561     default:
 562         return -2;
 563
 564     }
 565 }
 566
 567 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 568                              const char *type, const char *value)
 569 {
 570     if (!value) {
 571         RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
 572         return 0;
 573     }
 574     if (strcmp(type, "rsa_padding_mode") == 0) {
 575         int pm;
 576         if (strcmp(value, "pkcs1") == 0)
 577             pm = RSA_PKCS1_PADDING;
 578         else if (strcmp(value, "sslv23") == 0)
 579             pm = RSA_SSLV23_PADDING;
 580         else if (strcmp(value, "none") == 0)
 581             pm = RSA_NO_PADDING;
 582         else if (strcmp(value, "oeap") == 0)
 583             pm = RSA_PKCS1_OAEP_PADDING;
 584         else if (strcmp(value, "oaep") == 0)
 585             pm = RSA_PKCS1_OAEP_PADDING;
 586         else if (strcmp(value, "x931") == 0)
 587             pm = RSA_X931_PADDING;
 588         else if (strcmp(value, "pss") == 0)
 589             pm = RSA_PKCS1_PSS_PADDING;
 590         else {
 591             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_UNKNOWN_PADDING_TYPE);
 592             return -2;
 593         }
 594         return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 595     }
 596
 597     if (strcmp(type, "rsa_pss_saltlen") == 0) {
 598         int saltlen;
 599         saltlen = atoi(value);
 600         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 601     }
 602
 603     if (strcmp(type, "rsa_keygen_bits") == 0) {
 604         int nbits;
 605         nbits = atoi(value);
 606         return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 607     }
 608
 609     if (strcmp(type, "rsa_keygen_pubexp") == 0) {
 610         int ret;
 611         BIGNUM *pubexp = NULL;
 612         if (!BN_asc2bn(&pubexp, value))
 613             return 0;
 614         ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 615         if (ret <= 0)
 616             BN_free(pubexp);
 617         return ret;
 618     }
 619
 620     if (strcmp(type, "rsa_mgf1_md") == 0) {
 621         const EVP_MD *md;
 622         if ((md = EVP_get_digestbyname(value)) == NULL) {
 623             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
 624             return 0;
 625         }
 626         return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
 627     }
 628
 629     if (strcmp(type, "rsa_oaep_md") == 0) {
 630         const EVP_MD *md;
 631         if ((md = EVP_get_digestbyname(value)) == NULL) {
 632             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
 633             return 0;
 634         }
 635         return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
 636     }
 637     if (strcmp(type, "rsa_oaep_label") == 0) {
 638         unsigned char *lab;
 639         long lablen;
 640         int ret;
 641         lab = string_to_hex(value, &lablen);
 642         if (!lab)
 643             return 0;
 644         ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
 645         if (ret <= 0)
 646             OPENSSL_free(lab);
 647         return ret;
 648     }
 649
 650     return -2;
 651 }
 652
 653 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 654 {
 655     RSA *rsa = NULL;
 656     RSA_PKEY_CTX *rctx = ctx->data;
 657     BN_GENCB *pcb;
 658     int ret;
 659     if (rctx->pub_exp == NULL) {
 660         rctx->pub_exp = BN_new();
 661         if (rctx->pub_exp == NULL || !BN_set_word(rctx->pub_exp, RSA_F4))
 662             return 0;
 663     }
 664     rsa = RSA_new();
 665     if (rsa == NULL)
 666         return 0;
 667     if (ctx->pkey_gencb) {
 668         pcb = BN_GENCB_new();
 669         if (pcb == NULL) {
 670             RSA_free(rsa);
 671             return 0;
 672         }
 673         evp_pkey_set_cb_translate(pcb, ctx);
 674     } else
 675         pcb = NULL;
 676     ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 677     BN_GENCB_free(pcb);
 678     if (ret > 0)
 679         EVP_PKEY_assign_RSA(pkey, rsa);
 680     else
 681         RSA_free(rsa);
 682     return ret;
 683 }
 684
 685 const EVP_PKEY_METHOD rsa_pkey_meth = {
 686     EVP_PKEY_RSA,
 687     EVP_PKEY_FLAG_AUTOARGLEN,
 688     pkey_rsa_init,
 689     pkey_rsa_copy,
 690     pkey_rsa_cleanup,
 691
 692     0, 0,
 693
 694     0,
 695     pkey_rsa_keygen,
 696
 697     0,
 698     pkey_rsa_sign,
 699
 700     0,
 701     pkey_rsa_verify,
 702
 703     0,
 704     pkey_rsa_verifyrecover,
 705
 706     0, 0, 0, 0,
 707
 708     0,
 709     pkey_rsa_encrypt,
 710
 711     0,
 712     pkey_rsa_decrypt,
 713
 714     0, 0,
 715
 716     pkey_rsa_ctrl,
 717     pkey_rsa_ctrl_str
 718 };