From: Adhemerval Zanella Date: Mon, 21 Apr 2025 14:32:02 +0000 (-0300) Subject: math: Optimize double ilogb/llogb X-Git-Tag: glibc-2.42~181 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c4be334400508e6aa007f99ccc36c414f793c387;p=thirdparty%2Fglibc.git math: Optimize double ilogb/llogb It removes the wrapper by moving the error/EDOM handling to an out-of-line implementation (__math_invalid_i/__math_invalid_li). Also, __glibc_unlikely is used on errors case since it helps code generation on recent gcc. The code now builds to with gcc-14 on aarch64: 0000000000000000 <__ilogb>: 0: 9e660000 fmov x0, d0 4: d374f801 ubfx x1, x0, #52, #11 8: 340000e1 cbz w1, 24 <__ilogb+0x24> c: 510ffc20 sub w0, w1, #0x3ff 10: 711ffc3f cmp w1, #0x7ff 14: 54000040 b.eq 1c <__ilogb+0x1c> // b.none 18: d65f03c0 ret 1c: 12b00000 mov w0, #0x7fffffff // #2147483647 20: 14000000 b 0 <__math_invalid_i> 24: d374cc00 lsl x0, x0, #12 28: b40000a0 cbz x0, 3c <__ilogb+0x3c> 2c: dac01000 clz x0, x0 30: 12807fc1 mov w1, #0xfffffc01 // #-1023 34: 4b000020 sub w0, w1, w0 38: d65f03c0 ret 3c: 320107e0 mov w0, #0x80000001 // #-2147483647 40: 14000000 b 0 <__math_invalid_i> Some ABI requires additional adjustments: * i386 and m68k requires to use the template version, since both provide __ieee754_ilogb implementatations. * loongarch uses a custom implementation as well. * powerpc64le also has a custom implementation for POWER9, which is also used for float and float128 version. The generic e_ilogb.c implementation is moved on powerpc to keep the current code as-is. Checked on aarch64-linux-gnu and x86_64-linux-gnu. Reviewed-by: Wilco Dijkstra --- diff --git a/sysdeps/i386/fpu/math_err.c b/sysdeps/i386/fpu/math_err.c deleted file mode 100644 index 1cc89317007..00000000000 --- a/sysdeps/i386/fpu/math_err.c +++ /dev/null @@ -1 +0,0 @@ -/* Not needed. */ diff --git a/sysdeps/i386/fpu/w_ilogb.c b/sysdeps/i386/fpu/w_ilogb.c new file mode 100644 index 00000000000..9c262170216 --- /dev/null +++ b/sysdeps/i386/fpu/w_ilogb.c @@ -0,0 +1,2 @@ +#include +#include diff --git a/sysdeps/i386/fpu/w_llogb.c b/sysdeps/i386/fpu/w_llogb.c new file mode 100644 index 00000000000..5e8891a6682 --- /dev/null +++ b/sysdeps/i386/fpu/w_llogb.c @@ -0,0 +1,2 @@ +#include +#include diff --git a/sysdeps/ieee754/dbl-64/e_ilogb.c b/sysdeps/ieee754/dbl-64/e_ilogb.c index 89e74982661..1ea2f23e2fd 100644 --- a/sysdeps/ieee754/dbl-64/e_ilogb.c +++ b/sysdeps/ieee754/dbl-64/e_ilogb.c @@ -1,41 +1 @@ -/* Get integer exponent of a floating-point value. - Copyright (C) 1999-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include "math_config.h" - -int -__ieee754_ilogb (double x) -{ - uint64_t ux = asuint64 (x); - int ex = (ux & ~SIGN_MASK) >> MANTISSA_WIDTH; - if (ex == 0) /* zero or subnormal */ - { - /* Clear sign and exponent */ - ux <<= 12; - if (ux == 0) - return FP_ILOGB0; - /* subnormal */ - return -1023 - stdc_leading_zeros (ux); - } - if (ex == EXPONENT_MASK >> MANTISSA_WIDTH) /* NaN or Inf */ - return ux << 12 ? FP_ILOGBNAN : INT_MAX; - return ex - 1023; -} +/* ilogb is implemented at w_ilogb.c */ diff --git a/sysdeps/ieee754/dbl-64/math_config.h b/sysdeps/ieee754/dbl-64/math_config.h index 3382e385f9f..5766f6a1db9 100644 --- a/sysdeps/ieee754/dbl-64/math_config.h +++ b/sysdeps/ieee754/dbl-64/math_config.h @@ -164,6 +164,8 @@ attribute_hidden double __math_divzero (uint32_t); /* Invalid input unless it is a quiet NaN. */ attribute_hidden double __math_invalid (double); +attribute_hidden int __math_invalid_i (int); +attribute_hidden long int __math_invalid_li (long int); /* Error handling using output checking, only for errno setting. */ diff --git a/sysdeps/ieee754/dbl-64/math_err.c b/sysdeps/ieee754/dbl-64/math_err.c index 4a07fd5714d..b8c645a3df1 100644 --- a/sysdeps/ieee754/dbl-64/math_err.c +++ b/sysdeps/ieee754/dbl-64/math_err.c @@ -29,8 +29,24 @@ with_errno (double y, int e) errno = e; return y; } + +NOINLINE static int +with_errno_i (int y, int e) +{ + errno = e; + return y; +} + +NOINLINE static long int +with_errno_li (long int y, int e) +{ + errno = e; + return y; +} #else #define with_errno(x, e) (x) +#define with_errno_i(x, e) (x) +#define with_errno_li(x, e) (x) #endif attribute_hidden double @@ -83,6 +99,22 @@ __math_invalid (double x) return isnan (x) ? y : with_errno (y, EDOM); } +attribute_hidden int +__math_invalid_i (int r) +{ + double y = 0.0 / 0.0; + math_force_eval (y); + return with_errno_i (r, EDOM); +} + +attribute_hidden long int +__math_invalid_li (long int r) +{ + double y = 0.0 / 0.0; + math_force_eval (y); + return with_errno_li (r, EDOM); +} + /* Check result and set errno if necessary. */ attribute_hidden double diff --git a/sysdeps/ieee754/dbl-64/w_ilogb-impl.h b/sysdeps/ieee754/dbl-64/w_ilogb-impl.h new file mode 100644 index 00000000000..c91973507f2 --- /dev/null +++ b/sysdeps/ieee754/dbl-64/w_ilogb-impl.h @@ -0,0 +1,37 @@ +/* Get integer exponent of a floating-point value. + Copyright (C) 1999-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +static inline RET_TYPE +IMPL_NAME (double x) +{ + uint64_t ux = asuint64 (x); + int ex = (ux & ~SIGN_MASK) >> MANTISSA_WIDTH; + if (__glibc_unlikely (ex == 0)) /* zero or subnormal */ + { + /* Clear sign and exponent */ + ux <<= 12; + if (ux == 0) + return RET_INVALID (RET_LOGB0); + /* subnormal */ + return (RET_TYPE)-1023 - stdc_leading_zeros (ux); + } + if (__glibc_unlikely (ex == EXPONENT_MASK >> MANTISSA_WIDTH)) + /* NaN or Inf */ + return RET_INVALID (ux << 12 ? RET_LOGBNAN : RET_LOGMAX); + return ex - 1023; +} diff --git a/sysdeps/ieee754/dbl-64/w_ilogb.c b/sysdeps/ieee754/dbl-64/w_ilogb.c new file mode 100644 index 00000000000..e460f14dc9e --- /dev/null +++ b/sysdeps/ieee754/dbl-64/w_ilogb.c @@ -0,0 +1,52 @@ +/* Get integer exponent of a floating-point value. + Copyright (C) 1999-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include +#include +#include "math_config.h" + +#ifdef DEF_AS_LLOGB +# define DECL_NAME __llogb +# define FUNC_NAME llogb +# define RET_TYPE long int +# define RET_LOGB0 FP_LLOGB0 +# define RET_LOGBNAN FP_LLOGBNAN +# define RET_LOGMAX LONG_MAX +# define RET_INVALID __math_invalid_li +#else +# define DECL_NAME __ilogb +# define FUNC_NAME ilogb +# define RET_TYPE int +# define RET_LOGB0 FP_ILOGB0 +# define RET_LOGBNAN FP_ILOGBNAN +# define RET_LOGMAX INT_MAX +# define RET_INVALID __math_invalid_i +#endif +#define __IMPL_NAME(x,y) x ## _ ## y +#define _IMPL_NAME(x,y) __IMPL_NAME(x,y) +#define IMPL_NAME _IMPL_NAME(FUNC_NAME, impl) +#include + +RET_TYPE +DECL_NAME (double x) +{ + return IMPL_NAME (x); +} +libm_alias_double (DECL_NAME, FUNC_NAME) diff --git a/sysdeps/ieee754/dbl-64/w_llogb.c b/sysdeps/ieee754/dbl-64/w_llogb.c new file mode 100644 index 00000000000..c984cd151ae --- /dev/null +++ b/sysdeps/ieee754/dbl-64/w_llogb.c @@ -0,0 +1,2 @@ +#define DEF_AS_LLOGB +#include "w_ilogb.c" diff --git a/sysdeps/loongarch/fpu/e_ilogb.c b/sysdeps/loongarch/fpu/w_ilogb-impl.h similarity index 86% rename from sysdeps/loongarch/fpu/e_ilogb.c rename to sysdeps/loongarch/fpu/w_ilogb-impl.h index f21fa5c019c..19053731892 100644 --- a/sysdeps/loongarch/fpu/e_ilogb.c +++ b/sysdeps/loongarch/fpu/w_ilogb-impl.h @@ -1,4 +1,4 @@ -/* __ieee754_ilogb(). LoongArch version. +/* Get integer exponent of a floating-point value. LoongArch version. Copyright (C) 2022-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,20 +16,18 @@ License along with the GNU C Library; if not, see . */ -#define NO_MATH_REDIRECT -#include #include -int -__ieee754_ilogb (double x) +static inline RET_TYPE +IMPL_NAME (double x) { int x_cond; asm volatile ("fclass.d \t%0, %1" : "=f" (x_cond) : "f" (x)); if (__glibc_unlikely (x_cond & _FCLASS_ZERO)) - return FP_ILOGB0; + return RET_INVALID (RET_LOGB0); else if (__glibc_unlikely (x_cond & ( _FCLASS_NAN | _FCLASS_INF))) - return FP_ILOGBNAN; + return RET_INVALID (RET_LOGBNAN); else { asm volatile ("fabs.d \t%0, %1" : "=f" (x) : "f" (x)); diff --git a/sysdeps/m68k/m680x0/fpu/math_err.c b/sysdeps/m68k/m680x0/fpu/math_err.c deleted file mode 100644 index 1cc89317007..00000000000 --- a/sysdeps/m68k/m680x0/fpu/math_err.c +++ /dev/null @@ -1 +0,0 @@ -/* Not needed. */ diff --git a/sysdeps/m68k/m680x0/w_ilogb.c b/sysdeps/m68k/m680x0/w_ilogb.c new file mode 100644 index 00000000000..9c262170216 --- /dev/null +++ b/sysdeps/m68k/m680x0/w_ilogb.c @@ -0,0 +1,2 @@ +#include +#include diff --git a/sysdeps/m68k/m680x0/w_llogb.c b/sysdeps/m68k/m680x0/w_llogb.c new file mode 100644 index 00000000000..5e8891a6682 --- /dev/null +++ b/sysdeps/m68k/m680x0/w_llogb.c @@ -0,0 +1,2 @@ +#include +#include diff --git a/sysdeps/powerpc/powerpc64/le/fpu/e_ilogb.c b/sysdeps/powerpc/powerpc64/le/fpu/e_ilogb.c new file mode 100644 index 00000000000..89e74982661 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/le/fpu/e_ilogb.c @@ -0,0 +1,41 @@ +/* Get integer exponent of a floating-point value. + Copyright (C) 1999-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include +#include "math_config.h" + +int +__ieee754_ilogb (double x) +{ + uint64_t ux = asuint64 (x); + int ex = (ux & ~SIGN_MASK) >> MANTISSA_WIDTH; + if (ex == 0) /* zero or subnormal */ + { + /* Clear sign and exponent */ + ux <<= 12; + if (ux == 0) + return FP_ILOGB0; + /* subnormal */ + return -1023 - stdc_leading_zeros (ux); + } + if (ex == EXPONENT_MASK >> MANTISSA_WIDTH) /* NaN or Inf */ + return ux << 12 ? FP_ILOGBNAN : INT_MAX; + return ex - 1023; +} diff --git a/sysdeps/powerpc/powerpc64/le/fpu/w_ilogb.c b/sysdeps/powerpc/powerpc64/le/fpu/w_ilogb.c new file mode 100644 index 00000000000..9c262170216 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/le/fpu/w_ilogb.c @@ -0,0 +1,2 @@ +#include +#include diff --git a/sysdeps/powerpc/powerpc64/le/fpu/w_llogb.c b/sysdeps/powerpc/powerpc64/le/fpu/w_llogb.c new file mode 100644 index 00000000000..5e8891a6682 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/le/fpu/w_llogb.c @@ -0,0 +1,2 @@ +#include +#include