From: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Tue, 22 Apr 2025 18:59:02 +0000 (-0300)
Subject: math: Fix UB in float128 atan2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=61ad23190ea96a28afb5ee3ba30c3133d7bc62f7;p=thirdparty%2Fglibc.git

math: Fix UB in float128 atan2

UBSAN: Undefined behaviour in ../sysdeps/ieee754/float128/../ldbl-128/e_atan2l.c:68:9 unsigned integer overflow: 9223372036854775808 - 4611404543450677248 cannot be represened in type 'long int'
---

diff --git a/sysdeps/ieee754/ldbl-128/e_atan2l.c b/sysdeps/ieee754/ldbl-128/e_atan2l.c
index cdf08170f6..d2701084de 100644
--- a/sysdeps/ieee754/ldbl-128/e_atan2l.c
+++ b/sysdeps/ieee754/ldbl-128/e_atan2l.c
@@ -65,7 +65,7 @@ __ieee754_atan2l(_Float128 y, _Float128 x)
 	if(((ix|((lx|-lx)>>63))>0x7fff000000000000LL)||
 	   ((iy|((ly|-ly)>>63))>0x7fff000000000000LL))	/* x or y is NaN */
 	   return x+y;
-	if(((hx-0x3fff000000000000LL)|lx)==0) return __atanl(y);   /* x=1.0L */
+	if((((uint64_t)hx-0x3fff000000000000LL)|lx)==0) return __atanl(y);   /* x=1.0L */
 	m = ((hy>>63)&1)|((hx>>62)&2);	/* 2*sign(x)+sign(y) */
 
     /* when y = 0 */