x86: Fix UB in isnanl

Building with --enable-ubasn triggers:

$ math/test-ldouble-roundeven
UBSAN: Undefined behaviour in ../sysdeps/ieee754/ldbl-96/s_roundl.c:75:28 left shift of 1 by 31 cannot be represented in type 'int'

Also adds the inputs that triggers it on isnan testcase.

diff --git a/math/libm-test-isnan.inc b/math/libm-test-isnan.inc
index 980aa5119fd3b9f00d9ed3af025b73d30da4672f..72d2ad210ec77a40adea542fe8a5a80c3ba391de 100644 (file)
--- a/math/libm-test-isnan.inc
+++ b/math/libm-test-isnan.inc
@@ -23,6 +23,8 @@ static const struct test_f_i_data isnan_test_data[] =
     TEST_f_b (isnan, 0, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_f_b (isnan, minus_zero, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_f_b (isnan, 10, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_f_b (isnan, 4294967297, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_f_b (isnan, -4294967297, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_f_b (isnan, min_subnorm_value, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_f_b (isnan, -min_subnorm_value, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_f_b (isnan, min_value, 0, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),

diff --git a/sysdeps/x86/fpu/s_isnanl.c b/sysdeps/x86/fpu/s_isnanl.c
index 41484bf6544e2c9293b530eae0a98b74aa030266..69e34b8aaf316dbb0c2439e950c41b98529386f7 100644 (file)
--- a/sysdeps/x86/fpu/s_isnanl.c
+++ b/sysdeps/x86/fpu/s_isnanl.c
@@ -26,12 +26,13 @@ static char rcsid[] = "$NetBSD: $";
 
 int __isnanl(long double x)
 {
-       int32_t se,hx,lx,pn;
+       int se;
+       unsigned int hx,lx,pn;
        GET_LDOUBLE_WORDS(se,hx,lx,x);
        se = (se & 0x7fff) << 1;
        /* Detect pseudo-normal numbers, i.e. exponent is non-zero and the top
           bit of the significand is not set.   */
-       pn = (uint32_t)((~hx & 0x80000000) & (se|(-se)))>>31;
+       pn = ((~hx & 0x80000000) & (se|(-se)))>>31;
        /* Clear the significand bit when computing mantissa.  */
        lx |= hx & 0x7fffffff;
        se |= (uint32_t)(lx|(-lx))>>31;