]> git.ipfire.org Git - thirdparty/openssl.git/blob - crypto/rsa/rsa_eay.c
David Brumley <dbrumley@stanford.edu> noted and corrected a case in the
[thirdparty/openssl.git] / crypto / rsa / rsa_eay.c
1 /* crypto/rsa/rsa_eay.c */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3 * All rights reserved.
4 *
5 * This package is an SSL implementation written
6 * by Eric Young (eay@cryptsoft.com).
7 * The implementation was written so as to conform with Netscapes SSL.
8 *
9 * This library is free for commercial and non-commercial use as long as
10 * the following conditions are aheared to. The following conditions
11 * apply to all code found in this distribution, be it the RC4, RSA,
12 * lhash, DES, etc., code; not just the SSL code. The SSL documentation
13 * included with this distribution is covered by the same copyright terms
14 * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15 *
16 * Copyright remains Eric Young's, and as such any Copyright notices in
17 * the code are not to be removed.
18 * If this package is used in a product, Eric Young should be given attribution
19 * as the author of the parts of the library used.
20 * This can be in the form of a textual message at program startup or
21 * in documentation (online or textual) provided with the package.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 * 1. Redistributions of source code must retain the copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in the
30 * documentation and/or other materials provided with the distribution.
31 * 3. All advertising materials mentioning features or use of this software
32 * must display the following acknowledgement:
33 * "This product includes cryptographic software written by
34 * Eric Young (eay@cryptsoft.com)"
35 * The word 'cryptographic' can be left out if the rouines from the library
36 * being used are not cryptographic related :-).
37 * 4. If you include any Windows specific code (or a derivative thereof) from
38 * the apps directory (application code) you must include an acknowledgement:
39 * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40 *
41 * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 *
53 * The licence and distribution terms for any publically available version or
54 * derivative of this code cannot be changed. i.e. this code cannot simply be
55 * copied and put under another distribution licence
56 * [including the GNU Public Licence.]
57 */
58
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/bn.h>
62 #include <openssl/rsa.h>
63 #include <openssl/rand.h>
64 #ifndef OPENSSL_NO_ENGINE
65 #include <openssl/engine.h>
66 #endif
67
68 #ifndef RSA_NULL
69
70 static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
71 unsigned char *to, RSA *rsa,int padding);
72 static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
73 unsigned char *to, RSA *rsa,int padding);
74 static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
75 unsigned char *to, RSA *rsa,int padding);
76 static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
77 unsigned char *to, RSA *rsa,int padding);
78 static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa);
79 static int RSA_eay_init(RSA *rsa);
80 static int RSA_eay_finish(RSA *rsa);
81 static RSA_METHOD rsa_pkcs1_eay_meth={
82 "Eric Young's PKCS#1 RSA",
83 RSA_eay_public_encrypt,
84 RSA_eay_public_decrypt, /* signature verification */
85 RSA_eay_private_encrypt, /* signing */
86 RSA_eay_private_decrypt,
87 RSA_eay_mod_exp,
88 BN_mod_exp_mont, /* XXX probably we should not use Montgomery if e == 3 */
89 RSA_eay_init,
90 RSA_eay_finish,
91 0, /* flags */
92 NULL,
93 0, /* rsa_sign */
94 0, /* rsa_verify */
95 NULL /* rsa_keygen */
96 };
97
98 const RSA_METHOD *RSA_PKCS1_SSLeay(void)
99 {
100 return(&rsa_pkcs1_eay_meth);
101 }
102
103 /* Static helper to reduce oodles of code duplication. As a slight
104 * optimisation, the "MONT_HELPER() macro must be used as front-end to this
105 * function, to prevent unnecessary function calls - there is an initial test
106 * that is performed by the macro-generated code. */
107 static int rsa_eay_mont_helper(BN_MONT_CTX **ptr, const BIGNUM *modulus, BN_CTX *ctx)
108 {
109 BN_MONT_CTX *bn_mont_ctx;
110 if((bn_mont_ctx = BN_MONT_CTX_new()) == NULL)
111 return 0;
112 if(!BN_MONT_CTX_set(bn_mont_ctx, modulus, ctx))
113 {
114 BN_MONT_CTX_free(bn_mont_ctx);
115 return 0;
116 }
117 if (*ptr == NULL) /* other thread may have finished first */
118 {
119 CRYPTO_w_lock(CRYPTO_LOCK_RSA);
120 if (*ptr == NULL) /* check again in the lock to stop races */
121 {
122 *ptr = bn_mont_ctx;
123 bn_mont_ctx = NULL;
124 }
125 CRYPTO_w_unlock(CRYPTO_LOCK_RSA);
126 }
127 if (bn_mont_ctx)
128 BN_MONT_CTX_free(bn_mont_ctx);
129 return 1;
130 }
131 /* Usage example;
132 * MONT_HELPER(rsa, bn_ctx, p, rsa->flags & RSA_FLAG_CACHE_PRIVATE, goto err);
133 */
134 #define MONT_HELPER(rsa, ctx, m, pre_cond, err_instr) \
135 if((pre_cond) && ((rsa)->_method_mod_##m == NULL) && \
136 !rsa_eay_mont_helper(&((rsa)->_method_mod_##m), \
137 (rsa)->m, (ctx))) \
138 err_instr
139
140 static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
141 unsigned char *to, RSA *rsa, int padding)
142 {
143 BIGNUM f,ret;
144 int i,j,k,num=0,r= -1;
145 unsigned char *buf=NULL;
146 BN_CTX *ctx=NULL;
147
148 BN_init(&f);
149 BN_init(&ret);
150 if ((ctx=BN_CTX_new()) == NULL) goto err;
151 num=BN_num_bytes(rsa->n);
152 if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
153 {
154 RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
155 goto err;
156 }
157
158 switch (padding)
159 {
160 case RSA_PKCS1_PADDING:
161 i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
162 break;
163 #ifndef OPENSSL_NO_SHA
164 case RSA_PKCS1_OAEP_PADDING:
165 i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
166 break;
167 #endif
168 case RSA_SSLV23_PADDING:
169 i=RSA_padding_add_SSLv23(buf,num,from,flen);
170 break;
171 case RSA_NO_PADDING:
172 i=RSA_padding_add_none(buf,num,from,flen);
173 break;
174 default:
175 RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
176 goto err;
177 }
178 if (i <= 0) goto err;
179
180 if (BN_bin2bn(buf,num,&f) == NULL) goto err;
181
182 if (BN_ucmp(&f, rsa->n) >= 0)
183 {
184 /* usually the padding functions would catch this */
185 RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
186 goto err;
187 }
188
189 MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
190
191 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
192 rsa->_method_mod_n)) goto err;
193
194 /* put in leading 0 bytes if the number is less than the
195 * length of the modulus */
196 j=BN_num_bytes(&ret);
197 i=BN_bn2bin(&ret,&(to[num-j]));
198 for (k=0; k<(num-i); k++)
199 to[k]=0;
200
201 r=num;
202 err:
203 if (ctx != NULL) BN_CTX_free(ctx);
204 BN_clear_free(&f);
205 BN_clear_free(&ret);
206 if (buf != NULL)
207 {
208 OPENSSL_cleanse(buf,num);
209 OPENSSL_free(buf);
210 }
211 return(r);
212 }
213
214 /* signing */
215 static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
216 unsigned char *to, RSA *rsa, int padding)
217 {
218 BIGNUM f,ret;
219 int i,j,k,num=0,r= -1;
220 unsigned char *buf=NULL;
221 BN_CTX *ctx=NULL;
222
223 BN_init(&f);
224 BN_init(&ret);
225
226 if ((ctx=BN_CTX_new()) == NULL) goto err;
227 num=BN_num_bytes(rsa->n);
228 if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
229 {
230 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
231 goto err;
232 }
233
234 switch (padding)
235 {
236 case RSA_PKCS1_PADDING:
237 i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
238 break;
239 case RSA_NO_PADDING:
240 i=RSA_padding_add_none(buf,num,from,flen);
241 break;
242 case RSA_SSLV23_PADDING:
243 default:
244 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
245 goto err;
246 }
247 if (i <= 0) goto err;
248
249 if (BN_bin2bn(buf,num,&f) == NULL) goto err;
250
251 if (BN_ucmp(&f, rsa->n) >= 0)
252 {
253 /* usually the padding functions would catch this */
254 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
255 goto err;
256 }
257
258 if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
259 RSA_blinding_on(rsa,ctx);
260 if (rsa->flags & RSA_FLAG_BLINDING)
261 if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
262
263 if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
264 ((rsa->p != NULL) &&
265 (rsa->q != NULL) &&
266 (rsa->dmp1 != NULL) &&
267 (rsa->dmq1 != NULL) &&
268 (rsa->iqmp != NULL)) )
269 { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
270 else
271 {
272 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
273 }
274
275 if (rsa->flags & RSA_FLAG_BLINDING)
276 if (!BN_BLINDING_invert(&ret,rsa->blinding,ctx)) goto err;
277
278 /* put in leading 0 bytes if the number is less than the
279 * length of the modulus */
280 j=BN_num_bytes(&ret);
281 i=BN_bn2bin(&ret,&(to[num-j]));
282 for (k=0; k<(num-i); k++)
283 to[k]=0;
284
285 r=num;
286 err:
287 if (ctx != NULL) BN_CTX_free(ctx);
288 BN_clear_free(&ret);
289 BN_clear_free(&f);
290 if (buf != NULL)
291 {
292 OPENSSL_cleanse(buf,num);
293 OPENSSL_free(buf);
294 }
295 return(r);
296 }
297
298 static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
299 unsigned char *to, RSA *rsa, int padding)
300 {
301 BIGNUM f,ret;
302 int j,num=0,r= -1;
303 unsigned char *p;
304 unsigned char *buf=NULL;
305 BN_CTX *ctx=NULL;
306
307 BN_init(&f);
308 BN_init(&ret);
309 ctx=BN_CTX_new();
310 if (ctx == NULL) goto err;
311
312 num=BN_num_bytes(rsa->n);
313
314 if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
315 {
316 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
317 goto err;
318 }
319
320 /* This check was for equality but PGP does evil things
321 * and chops off the top '0' bytes */
322 if (flen > num)
323 {
324 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
325 goto err;
326 }
327
328 /* make data into a big number */
329 if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;
330
331 if (BN_ucmp(&f, rsa->n) >= 0)
332 {
333 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
334 goto err;
335 }
336
337 if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
338 RSA_blinding_on(rsa,ctx);
339 if (rsa->flags & RSA_FLAG_BLINDING)
340 if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
341
342 /* do the decrypt */
343 if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
344 ((rsa->p != NULL) &&
345 (rsa->q != NULL) &&
346 (rsa->dmp1 != NULL) &&
347 (rsa->dmq1 != NULL) &&
348 (rsa->iqmp != NULL)) )
349 { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
350 else
351 {
352 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
353 goto err;
354 }
355
356 if (rsa->flags & RSA_FLAG_BLINDING)
357 if (!BN_BLINDING_invert(&ret,rsa->blinding,ctx)) goto err;
358
359 p=buf;
360 j=BN_bn2bin(&ret,p); /* j is only used with no-padding mode */
361
362 switch (padding)
363 {
364 case RSA_PKCS1_PADDING:
365 r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
366 break;
367 #ifndef OPENSSL_NO_SHA
368 case RSA_PKCS1_OAEP_PADDING:
369 r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
370 break;
371 #endif
372 case RSA_SSLV23_PADDING:
373 r=RSA_padding_check_SSLv23(to,num,buf,j,num);
374 break;
375 case RSA_NO_PADDING:
376 r=RSA_padding_check_none(to,num,buf,j,num);
377 break;
378 default:
379 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
380 goto err;
381 }
382 if (r < 0)
383 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
384
385 err:
386 if (ctx != NULL) BN_CTX_free(ctx);
387 BN_clear_free(&f);
388 BN_clear_free(&ret);
389 if (buf != NULL)
390 {
391 OPENSSL_cleanse(buf,num);
392 OPENSSL_free(buf);
393 }
394 return(r);
395 }
396
397 /* signature verification */
398 static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
399 unsigned char *to, RSA *rsa, int padding)
400 {
401 BIGNUM f,ret;
402 int i,num=0,r= -1;
403 unsigned char *p;
404 unsigned char *buf=NULL;
405 BN_CTX *ctx=NULL;
406
407 BN_init(&f);
408 BN_init(&ret);
409 ctx=BN_CTX_new();
410 if (ctx == NULL) goto err;
411
412 num=BN_num_bytes(rsa->n);
413 buf=(unsigned char *)OPENSSL_malloc(num);
414 if (buf == NULL)
415 {
416 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
417 goto err;
418 }
419
420 /* This check was for equality but PGP does evil things
421 * and chops off the top '0' bytes */
422 if (flen > num)
423 {
424 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
425 goto err;
426 }
427
428 if (BN_bin2bn(from,flen,&f) == NULL) goto err;
429
430 if (BN_ucmp(&f, rsa->n) >= 0)
431 {
432 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
433 goto err;
434 }
435
436 MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
437
438 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
439 rsa->_method_mod_n)) goto err;
440
441 p=buf;
442 i=BN_bn2bin(&ret,p);
443
444 switch (padding)
445 {
446 case RSA_PKCS1_PADDING:
447 r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
448 break;
449 case RSA_NO_PADDING:
450 r=RSA_padding_check_none(to,num,buf,i,num);
451 break;
452 default:
453 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
454 goto err;
455 }
456 if (r < 0)
457 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
458
459 err:
460 if (ctx != NULL) BN_CTX_free(ctx);
461 BN_clear_free(&f);
462 BN_clear_free(&ret);
463 if (buf != NULL)
464 {
465 OPENSSL_cleanse(buf,num);
466 OPENSSL_free(buf);
467 }
468 return(r);
469 }
470
471 static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
472 {
473 BIGNUM r1,m1,vrfy;
474 int ret=0;
475 BN_CTX *ctx;
476
477 BN_init(&m1);
478 BN_init(&r1);
479 BN_init(&vrfy);
480 if ((ctx=BN_CTX_new()) == NULL) goto err;
481
482 MONT_HELPER(rsa, ctx, p, rsa->flags & RSA_FLAG_CACHE_PRIVATE, goto err);
483 MONT_HELPER(rsa, ctx, q, rsa->flags & RSA_FLAG_CACHE_PRIVATE, goto err);
484 MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PRIVATE, goto err);
485
486 if (!BN_mod(&r1,I,rsa->q,ctx)) goto err;
487 if (!rsa->meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
488 rsa->_method_mod_q)) goto err;
489
490 if (!BN_mod(&r1,I,rsa->p,ctx)) goto err;
491 if (!rsa->meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
492 rsa->_method_mod_p)) goto err;
493
494 if (!BN_sub(r0,r0,&m1)) goto err;
495 /* This will help stop the size of r0 increasing, which does
496 * affect the multiply if it optimised for a power of 2 size */
497 if (BN_get_sign(r0))
498 if (!BN_add(r0,r0,rsa->p)) goto err;
499
500 if (!BN_mul(&r1,r0,rsa->iqmp,ctx)) goto err;
501 if (!BN_mod(r0,&r1,rsa->p,ctx)) goto err;
502 /* If p < q it is occasionally possible for the correction of
503 * adding 'p' if r0 is negative above to leave the result still
504 * negative. This can break the private key operations: the following
505 * second correction should *always* correct this rare occurrence.
506 * This will *never* happen with OpenSSL generated keys because
507 * they ensure p > q [steve]
508 */
509 if (BN_get_sign(r0))
510 if (!BN_add(r0,r0,rsa->p)) goto err;
511 if (!BN_mul(&r1,r0,rsa->q,ctx)) goto err;
512 if (!BN_add(r0,&r1,&m1)) goto err;
513
514 if (rsa->e && rsa->n)
515 {
516 if (!rsa->meth->bn_mod_exp(&vrfy,r0,rsa->e,rsa->n,ctx,rsa->_method_mod_n)) goto err;
517 /* If 'I' was greater than (or equal to) rsa->n, the operation
518 * will be equivalent to using 'I mod n'. However, the result of
519 * the verify will *always* be less than 'n' so we don't check
520 * for absolute equality, just congruency. */
521 if (!BN_sub(&vrfy, &vrfy, I)) goto err;
522 if (!BN_mod(&vrfy, &vrfy, rsa->n, ctx)) goto err;
523 if (BN_get_sign(&vrfy))
524 if (!BN_add(&vrfy, &vrfy, rsa->n)) goto err;
525 if (!BN_is_zero(&vrfy))
526 /* 'I' and 'vrfy' aren't congruent mod n. Don't leak
527 * miscalculated CRT output, just do a raw (slower)
528 * mod_exp and return that instead. */
529 if (!rsa->meth->bn_mod_exp(r0,I,rsa->d,rsa->n,ctx,NULL)) goto err;
530 }
531 ret=1;
532 err:
533 BN_clear_free(&m1);
534 BN_clear_free(&r1);
535 BN_clear_free(&vrfy);
536 BN_CTX_free(ctx);
537 return(ret);
538 }
539
540 static int RSA_eay_init(RSA *rsa)
541 {
542 rsa->flags|=RSA_FLAG_CACHE_PUBLIC|RSA_FLAG_CACHE_PRIVATE;
543 return(1);
544 }
545
546 static int RSA_eay_finish(RSA *rsa)
547 {
548 if (rsa->_method_mod_n != NULL)
549 BN_MONT_CTX_free(rsa->_method_mod_n);
550 if (rsa->_method_mod_p != NULL)
551 BN_MONT_CTX_free(rsa->_method_mod_p);
552 if (rsa->_method_mod_q != NULL)
553 BN_MONT_CTX_free(rsa->_method_mod_q);
554 return(1);
555 }
556
557 #endif