From 748e85308ef4f3e672975b3604ea2d76424fa404 Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Fri, 24 Jun 2016 11:05:48 -0400 Subject: [PATCH] Fix BN_is_prime* calls. This function returns a tri-state -1 on error. See BoringSSL's 53409ee3d7595ed37da472bc73b010cd2c8a5ffd. Signed-off-by: Kurt Roeckx Reviewed-by: Rich Salz GH: #1251 --- apps/s_client.c | 4 ++-- crypto/bn/bn_x931p.c | 17 ++++++++++++----- crypto/dh/dh_check.c | 17 +++++++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/apps/s_client.c b/apps/s_client.c index 56a7081cab..e79cf7e496 100644 --- a/apps/s_client.c +++ b/apps/s_client.c @@ -249,10 +249,10 @@ static int srp_Verify_N_and_g(const BIGNUM *N, const BIGNUM *g) BIGNUM *r = BN_new(); int ret = g != NULL && N != NULL && bn_ctx != NULL && BN_is_odd(N) && - BN_is_prime_ex(N, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) && + BN_is_prime_ex(N, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) == 1 && p != NULL && BN_rshift1(p, N) && /* p = (N-1)/2 */ - BN_is_prime_ex(p, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) && + BN_is_prime_ex(p, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) == 1 && r != NULL && /* verify g^((N-1)/2) == -1 (mod N) */ BN_mod_exp(r, g, p, N, bn_ctx) && diff --git a/crypto/bn/bn_x931p.c b/crypto/bn/bn_x931p.c index 83170d4919..d863386ef4 100644 --- a/crypto/bn/bn_x931p.c +++ b/crypto/bn/bn_x931p.c @@ -21,7 +21,7 @@ static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb) { - int i = 0; + int i = 0, is_prime; if (!BN_copy(pi, Xpi)) return 0; if (!BN_is_odd(pi) && !BN_add_word(pi, 1)) @@ -30,7 +30,10 @@ static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, i++; BN_GENCB_call(cb, 0, i); /* NB 27 MR is specified in X9.31 */ - if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb)) + is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb); + if (is_prime < 0) + return 0; + if (is_prime) break; if (!BN_add_word(pi, 2)) return 0; @@ -119,14 +122,18 @@ int BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2, goto err; if (!BN_gcd(t, pm1, e, ctx)) goto err; - if (BN_is_one(t) + if (BN_is_one(t)) { /* * X9.31 specifies 8 MR and 1 Lucas test or any prime test * offering similar or better guarantees 50 MR is considerably * better. */ - && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb)) - break; + int r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb); + if (r < 0) + goto err; + if (r) + break; + } if (!BN_add(p, p, p1p2)) goto err; } diff --git a/crypto/dh/dh_check.c b/crypto/dh/dh_check.c index 523e31d05e..8d2e096c08 100644 --- a/crypto/dh/dh_check.c +++ b/crypto/dh/dh_check.c @@ -24,7 +24,7 @@ int DH_check(const DH *dh, int *ret) { - int ok = 0; + int ok = 0, r; BN_CTX *ctx = NULL; BN_ULONG l; BIGNUM *t1 = NULL, *t2 = NULL; @@ -53,7 +53,10 @@ int DH_check(const DH *dh, int *ret) if (!BN_is_one(t1)) *ret |= DH_NOT_SUITABLE_GENERATOR; } - if (!BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL)) + r = BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL); + if (r < 0) + goto err; + if (!r) *ret |= DH_CHECK_Q_NOT_PRIME; /* Check p == 1 mod q i.e. q divides p - 1 */ if (!BN_div(t1, t2, dh->p, dh->q, ctx)) @@ -74,12 +77,18 @@ int DH_check(const DH *dh, int *ret) } else *ret |= DH_UNABLE_TO_CHECK_GENERATOR; - if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL)) + r = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL); + if (r < 0) + goto err; + if (!r) *ret |= DH_CHECK_P_NOT_PRIME; else if (!dh->q) { if (!BN_rshift1(t1, dh->p)) goto err; - if (!BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL)) + r = BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL); + if (r < 0) + goto err; + if (!r) *ret |= DH_CHECK_P_NOT_SAFE_PRIME; } ok = 1; -- 2.39.2