Fix BN_gcd errors for some curves

Those even order that do not play nicely with Montgomery arithmetic

(back-ported from commit 3a6a4a93518fbb3d96632bfdcb538d340f29c56b)

Reviewed-by: Andy Polyakov <appro@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6810)

diff --git a/crypto/ec/ec_lib.c b/crypto/ec/ec_lib.c
index 3241aa51d9f47ae6db8d5615b619a0df2ceb3dfb..0890109980e40674f56d90d087688bd029967964 100644 (file)
--- a/crypto/ec/ec_lib.c
+++ b/crypto/ec/ec_lib.c
@@ -319,12 +319,16 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator,
         BN_zero(&group->cofactor);
 
     /*
-     * We ignore the return value because some groups have an order with
+     * Some groups have an order with
      * factors of two, which makes the Montgomery setup fail.
      * |group->mont_data| will be NULL in this case.
      */
-    ec_precompute_mont_data(group);
+    if (BN_is_odd(&group->order)) {
+        return ec_precompute_mont_data(group);
+    }
 
+    BN_MONT_CTX_free(group->mont_data);
+    group->mont_data = NULL;
     return 1;
 }