BN_gcd(): Avoid shifts of negative values

Fixes #22216

Thanks to Leland Mills for investigation and testing.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22272)

diff --git a/crypto/bn/bn_gcd.c b/crypto/bn/bn_gcd.c
index 519bb4e951d7dc019d3500b74ab95acd01a93182..2cd8ee35e034a38f1200e25a38d5dfb180cd0189 100644 (file)
--- a/crypto/bn/bn_gcd.c
+++ b/crypto/bn/bn_gcd.c
@@ -642,9 +642,9 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
 
     for (i = 0; i < m; i++) {
         /* conditionally flip signs if delta is positive and g is odd */
-        cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
+        cond = ((unsigned int)-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
             /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
-            & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1)));
+            & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1)));
         delta = (-cond & -delta) | ((cond - 1) & delta);
         r->neg ^= cond;
         /* swap */
@@ -656,7 +656,7 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
             goto err;
         BN_consttime_swap(g->d[0] & 1 /* g is odd */
                 /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
-                & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))),
+                & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1))),
                 g, temp, top);
         if (!BN_rshift1(g, g))
             goto err;