math: Fix UB in float128 atan2

UBSAN: Undefined behaviour in ../sysdeps/ieee754/float128/../ldbl-128/e_atan2l.c:68:9 unsigned integer overflow: 9223372036854775808 - 4611404543450677248 cannot be represened in type 'long int'

diff --git a/sysdeps/ieee754/ldbl-128/e_atan2l.c b/sysdeps/ieee754/ldbl-128/e_atan2l.c
index cdf08170f69ff4930766e6e730e398a4e904ce40..d2701084de993d768668a6c5113d451e5e6bf5e2 100644 (file)
--- a/sysdeps/ieee754/ldbl-128/e_atan2l.c
+++ b/sysdeps/ieee754/ldbl-128/e_atan2l.c
@@ -65,7 +65,7 @@ __ieee754_atan2l(_Float128 y, _Float128 x)
        if(((ix|((lx|-lx)>>63))>0x7fff000000000000LL)||
           ((iy|((ly|-ly)>>63))>0x7fff000000000000LL))  /* x or y is NaN */
           return x+y;
-       if(((hx-0x3fff000000000000LL)|lx)==0) return __atanl(y);   /* x=1.0L */
+       if((((uint64_t)hx-0x3fff000000000000LL)|lx)==0) return __atanl(y);   /* x=1.0L */
        m = ((hy>>63)&1)|((hx>>62)&2);  /* 2*sign(x)+sign(y) */
 
     /* when y = 0 */