sysdeps/x86/fpu/bits/mathinline.h

   1 /* Inline math functions for i387 and SSE.
   2    Copyright (C) 1995-2017 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _MATH_H
  20 # error "Never use <bits/mathinline.h> directly; include <math.h> instead."
  21 #endif
  22
  23 #ifndef __extern_always_inline
  24 # define __MATH_INLINE __inline
  25 #else
  26 # define __MATH_INLINE __extern_always_inline
  27 #endif
  28
  29 /* The gcc, version 2.7 or below, has problems with all this inlining
  30    code.  So disable it for this version of the compiler.  */
  31 # if __GNUC_PREREQ (2, 8)
  32
  33 /* Test for negative number.  Used in the signbit() macro.  */
  34 __MATH_INLINE int
  35 __NTH (__signbitf (float __x))
  36 {
  37 #  ifdef __SSE2_MATH__
  38   int __m;
  39   __asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
  40   return (__m & 0x8) != 0;
  41 #  else
  42   __extension__ union { float __f; int __i; } __u = { __f: __x };
  43   return __u.__i < 0;
  44 #  endif
  45 }
  46 __MATH_INLINE int
  47 __NTH (__signbit (double __x))
  48 {
  49 #  ifdef __SSE2_MATH__
  50   int __m;
  51   __asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
  52   return (__m & 0x80) != 0;
  53 #  else
  54   __extension__ union { double __d; int __i[2]; } __u = { __d: __x };
  55   return __u.__i[1] < 0;
  56 #  endif
  57 }
  58 __MATH_INLINE int
  59 __NTH (__signbitl (long double __x))
  60 {
  61   __extension__ union { long double __l; int __i[3]; } __u = { __l: __x };
  62   return (__u.__i[2] & 0x8000) != 0;
  63 }
  64
  65 # endif
  66
  67 /* The gcc, version 2.7 or below, has problems with all this inlining
  68    code.  So disable it for this version of the compiler.  */
  69 #if __GNUC_PREREQ (2, 8)
  70 # if !__GNUC_PREREQ (3, 4) && !defined __NO_MATH_INLINES \
  71      && defined __OPTIMIZE__
  72 /* GCC 3.4 introduced builtins for all functions below, so
  73    there's no need to define any of these inline functions.  */
  74
  75 #  ifdef __USE_ISOC99
  76
  77 /* Round to nearest integer.  */
  78 #   ifdef __SSE_MATH__
  79 __MATH_INLINE long int
  80 __NTH (lrintf (float __x))
  81 {
  82   long int __res;
  83   /* Mark as volatile since the result is dependent on the state of
  84      the SSE control register (the rounding mode).  Otherwise GCC might
  85      remove these assembler instructions since it does not know about
  86      the rounding mode change and cannot currently be told.  */
  87   __asm __volatile__ ("cvtss2si %1, %0" : "=r" (__res) : "xm" (__x));
  88   return __res;
  89 }
  90 #   endif
  91 #   ifdef __SSE2_MATH__
  92 __MATH_INLINE long int
  93 __NTH (lrint (double __x))
  94 {
  95   long int __res;
  96   /* Mark as volatile since the result is dependent on the state of
  97      the SSE control register (the rounding mode).  Otherwise GCC might
  98      remove these assembler instructions since it does not know about
  99      the rounding mode change and cannot currently be told.  */
 100   __asm __volatile__ ("cvtsd2si %1, %0" : "=r" (__res) : "xm" (__x));
 101   return __res;
 102 }
 103 #   endif
 104 #   ifdef __x86_64__
 105 __extension__
 106 __MATH_INLINE long long int
 107 __NTH (llrintf (float __x))
 108 {
 109   long long int __res;
 110   /* Mark as volatile since the result is dependent on the state of
 111      the SSE control register (the rounding mode).  Otherwise GCC might
 112      remove these assembler instructions since it does not know about
 113      the rounding mode change and cannot currently be told.  */
 114   __asm __volatile__ ("cvtss2si %1, %0" : "=r" (__res) : "xm" (__x));
 115   return __res;
 116 }
 117 __extension__
 118 __MATH_INLINE long long int
 119 __NTH (llrint (double __x))
 120 {
 121   long long int __res;
 122   /* Mark as volatile since the result is dependent on the state of
 123      the SSE control register (the rounding mode).  Otherwise GCC might
 124      remove these assembler instructions since it does not know about
 125      the rounding mode change and cannot currently be told.  */
 126   __asm __volatile__ ("cvtsd2si %1, %0" : "=r" (__res) : "xm" (__x));
 127   return __res;
 128 }
 129 #   endif
 130
 131 #   if defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ > 0 \
 132        && defined __SSE2_MATH__
 133 /* Determine maximum of two values.  */
 134 __MATH_INLINE float
 135 __NTH (fmaxf (float __x, float __y))
 136 {
 137 #    ifdef __AVX__
 138   float __res;
 139   __asm ("vmaxss %2, %1, %0" : "=x" (__res) : "x" (x), "xm" (__y));
 140   return __res;
 141 #    else
 142   __asm ("maxss %1, %0" : "+x" (__x) : "xm" (__y));
 143   return __x;
 144 #    endif
 145 }
 146 __MATH_INLINE double
 147 __NTH (fmax (double __x, double __y))
 148 {
 149 #    ifdef __AVX__
 150   float __res;
 151   __asm ("vmaxsd %2, %1, %0" : "=x" (__res) : "x" (x), "xm" (__y));
 152   return __res;
 153 #    else
 154   __asm ("maxsd %1, %0" : "+x" (__x) : "xm" (__y));
 155   return __x;
 156 #    endif
 157 }
 158
 159 /* Determine minimum of two values.  */
 160 __MATH_INLINE float
 161 __NTH (fminf (float __x, float __y))
 162 {
 163 #    ifdef __AVX__
 164   float __res;
 165   __asm ("vminss %2, %1, %0" : "=x" (__res) : "x" (x), "xm" (__y));
 166   return __res;
 167 #    else
 168   __asm ("minss %1, %0" : "+x" (__x) : "xm" (__y));
 169   return __x;
 170 #    endif
 171 }
 172 __MATH_INLINE double
 173 __NTH (fmin (double __x, double __y))
 174 {
 175 #    ifdef __AVX__
 176   float __res;
 177   __asm ("vminsd %2, %1, %0" : "=x" (__res) : "x" (x), "xm" (__y));
 178   return __res;
 179 #    else
 180   __asm ("minsd %1, %0" : "+x" (__x) : "xm" (__y));
 181   return __x;
 182 #    endif
 183 }
 184 #   endif
 185
 186 #  endif
 187
 188 #  if defined __SSE4_1__ && defined __SSE2_MATH__
 189 #   if defined __USE_XOPEN_EXTENDED || defined __USE_ISOC99
 190
 191 /* Round to nearest integer.  */
 192 __MATH_INLINE double
 193 __NTH (rint (double __x))
 194 {
 195   double __res;
 196   /* Mark as volatile since the result is dependent on the state of
 197      the SSE control register (the rounding mode).  Otherwise GCC might
 198      remove these assembler instructions since it does not know about
 199      the rounding mode change and cannot currently be told.  */
 200   __asm __volatile__ ("roundsd $4, %1, %0" : "=x" (__res) : "xm" (__x));
 201   return __res;
 202 }
 203 __MATH_INLINE float
 204 __NTH (rintf (float __x))
 205 {
 206   float __res;
 207   /* Mark as volatile since the result is dependent on the state of
 208      the SSE control register (the rounding mode).  Otherwise GCC might
 209      remove these assembler instructions since it does not know about
 210      the rounding mode change and cannot currently be told.  */
 211   __asm __volatile__ ("roundss $4, %1, %0" : "=x" (__res) : "xm" (__x));
 212   return __res;
 213 }
 214
 215 #    ifdef __USE_ISOC99
 216 /* Round to nearest integer without raising inexact exception.  */
 217 __MATH_INLINE double
 218 __NTH (nearbyint (double __x))
 219 {
 220   double __res;
 221   /* Mark as volatile since the result is dependent on the state of
 222      the SSE control register (the rounding mode).  Otherwise GCC might
 223      remove these assembler instructions since it does not know about
 224      the rounding mode change and cannot currently be told.  */
 225   __asm __volatile__ ("roundsd $0xc, %1, %0" : "=x" (__res) : "xm" (__x));
 226   return __res;
 227 }
 228 __MATH_INLINE float
 229 __NTH (nearbyintf (float __x))
 230 {
 231   float __res;
 232   /* Mark as volatile since the result is dependent on the state of
 233      the SSE control register (the rounding mode).  Otherwise GCC might
 234      remove these assembler instructions since it does not know about
 235      the rounding mode change and cannot currently be told.  */
 236   __asm __volatile__ ("roundss $0xc, %1, %0" : "=x" (__res) : "xm" (__x));
 237   return __res;
 238 }
 239 #    endif
 240
 241 #   endif
 242
 243 /* Smallest integral value not less than X.  */
 244 __MATH_INLINE double
 245 __NTH (ceil (double __x))
 246 {
 247   double __res;
 248   __asm ("roundsd $2, %1, %0" : "=x" (__res) : "xm" (__x));
 249   return __res;
 250 }
 251
 252 __MATH_INLINE float
 253 __NTH (ceilf (float __x))
 254 {
 255   float __res;
 256   __asm ("roundss $2, %1, %0" : "=x" (__res) : "xm" (__x));
 257   return __res;
 258 }
 259
 260 /* Largest integer not greater than X.  */
 261 __MATH_INLINE double
 262 __NTH (floor (double __x))
 263 {
 264   double __res;
 265   __asm ("roundsd $1, %1, %0" : "=x" (__res) : "xm" (__x));
 266   return __res;
 267 }
 268
 269 __MATH_INLINE float
 270 __NTH (floorf (float __x))
 271 {
 272   float __res;
 273   __asm ("roundss $1, %1, %0" : "=x" (__res) : "xm" (__x));
 274   return __res;
 275 }
 276 #  endif
 277 # endif
 278 #endif
 279
 280 /* Disable x87 inlines when -fpmath=sse is passed and also when we're building
 281    on x86_64.  Older gcc (gcc-3.2 for example) does not define __SSE2_MATH__
 282    for x86_64.  */
 283 #if !defined __SSE2_MATH__ && !defined __x86_64__
 284 # if ((!defined __NO_MATH_INLINES || defined __LIBC_INTERNAL_MATH_INLINES) \
 285      && defined __OPTIMIZE__)
 286
 287 /* The inline functions do not set errno or raise necessarily the
 288    correct exceptions.  */
 289 #  undef math_errhandling
 290
 291 /* A macro to define float, double, and long double versions of various
 292    math functions for the ix87 FPU.  FUNC is the function name (which will
 293    be suffixed with f and l for the float and long double version,
 294    respectively).  OP is the name of the FPU operation.
 295    We define two sets of macros.  The set with the additional NP
 296    doesn't add a prototype declaration.  */
 297
 298 #  ifdef __USE_ISOC99
 299 #   define __inline_mathop(func, op) \
 300   __inline_mathop_ (double, func, op)                                         \
 301   __inline_mathop_ (float, __CONCAT(func,f), op)                              \
 302   __inline_mathop_ (long double, __CONCAT(func,l), op)
 303 #   define __inline_mathopNP(func, op) \
 304   __inline_mathopNP_ (double, func, op)                                       \
 305   __inline_mathopNP_ (float, __CONCAT(func,f), op)                            \
 306   __inline_mathopNP_ (long double, __CONCAT(func,l), op)
 307 #  else
 308 #   define __inline_mathop(func, op) \
 309   __inline_mathop_ (double, func, op)
 310 #   define __inline_mathopNP(func, op) \
 311   __inline_mathopNP_ (double, func, op)
 312 #  endif
 313
 314 #  define __inline_mathop_(float_type, func, op) \
 315   __inline_mathop_decl_ (float_type, func, op, "0" (__x))
 316 #  define __inline_mathopNP_(float_type, func, op) \
 317   __inline_mathop_declNP_ (float_type, func, op, "0" (__x))
 318
 319
 320 #  ifdef __USE_ISOC99
 321 #   define __inline_mathop_decl(func, op, params...) \
 322   __inline_mathop_decl_ (double, func, op, params)                            \
 323   __inline_mathop_decl_ (float, __CONCAT(func,f), op, params)                 \
 324   __inline_mathop_decl_ (long double, __CONCAT(func,l), op, params)
 325 #   define __inline_mathop_declNP(func, op, params...) \
 326   __inline_mathop_declNP_ (double, func, op, params)                          \
 327   __inline_mathop_declNP_ (float, __CONCAT(func,f), op, params)               \
 328   __inline_mathop_declNP_ (long double, __CONCAT(func,l), op, params)
 329 #  else
 330 #   define __inline_mathop_decl(func, op, params...) \
 331   __inline_mathop_decl_ (double, func, op, params)
 332 #   define __inline_mathop_declNP(func, op, params...) \
 333   __inline_mathop_declNP_ (double, func, op, params)
 334 #  endif
 335
 336 #  define __inline_mathop_decl_(float_type, func, op, params...) \
 337   __MATH_INLINE float_type func (float_type) __THROW;                         \
 338   __inline_mathop_declNP_ (float_type, func, op, params)
 339
 340 #  define __inline_mathop_declNP_(float_type, func, op, params...) \
 341   __MATH_INLINE float_type __NTH (func (float_type __x))                      \
 342   {                                                                           \
 343     register float_type __result;                                             \
 344     __asm __volatile__ (op : "=t" (__result) : params);                       \
 345     return __result;                                                          \
 346   }
 347
 348
 349 #  ifdef __USE_ISOC99
 350 #   define __inline_mathcode(func, arg, code) \
 351   __inline_mathcode_ (double, func, arg, code)                                \
 352   __inline_mathcode_ (float, __CONCAT(func,f), arg, code)                     \
 353   __inline_mathcode_ (long double, __CONCAT(func,l), arg, code)
 354 #   define __inline_mathcodeNP(func, arg, code) \
 355   __inline_mathcodeNP_ (double, func, arg, code)                              \
 356   __inline_mathcodeNP_ (float, __CONCAT(func,f), arg, code)                   \
 357   __inline_mathcodeNP_ (long double, __CONCAT(func,l), arg, code)
 358 #   define __inline_mathcode2(func, arg1, arg2, code) \
 359   __inline_mathcode2_ (double, func, arg1, arg2, code)                        \
 360   __inline_mathcode2_ (float, __CONCAT(func,f), arg1, arg2, code)             \
 361   __inline_mathcode2_ (long double, __CONCAT(func,l), arg1, arg2, code)
 362 #   define __inline_mathcodeNP2(func, arg1, arg2, code) \
 363   __inline_mathcodeNP2_ (double, func, arg1, arg2, code)                      \
 364   __inline_mathcodeNP2_ (float, __CONCAT(func,f), arg1, arg2, code)           \
 365   __inline_mathcodeNP2_ (long double, __CONCAT(func,l), arg1, arg2, code)
 366 #   define __inline_mathcode3(func, arg1, arg2, arg3, code) \
 367   __inline_mathcode3_ (double, func, arg1, arg2, arg3, code)                  \
 368   __inline_mathcode3_ (float, __CONCAT(func,f), arg1, arg2, arg3, code)       \
 369   __inline_mathcode3_ (long double, __CONCAT(func,l), arg1, arg2, arg3, code)
 370 #   define __inline_mathcodeNP3(func, arg1, arg2, arg3, code) \
 371   __inline_mathcodeNP3_ (double, func, arg1, arg2, arg3, code)                \
 372   __inline_mathcodeNP3_ (float, __CONCAT(func,f), arg1, arg2, arg3, code)     \
 373   __inline_mathcodeNP3_ (long double, __CONCAT(func,l), arg1, arg2, arg3, code)
 374 #  else
 375 #   define __inline_mathcode(func, arg, code) \
 376   __inline_mathcode_ (double, func, (arg), code)
 377 #   define __inline_mathcodeNP(func, arg, code) \
 378   __inline_mathcodeNP_ (double, func, (arg), code)
 379 #   define __inline_mathcode2(func, arg1, arg2, code) \
 380   __inline_mathcode2_ (double, func, arg1, arg2, code)
 381 #   define __inline_mathcodeNP2(func, arg1, arg2, code) \
 382   __inline_mathcodeNP2_ (double, func, arg1, arg2, code)
 383 #   define __inline_mathcode3(func, arg1, arg2, arg3, code) \
 384   __inline_mathcode3_ (double, func, arg1, arg2, arg3, code)
 385 #   define __inline_mathcodeNP3(func, arg1, arg2, arg3, code) \
 386   __inline_mathcodeNP3_ (double, func, arg1, arg2, arg3, code)
 387 #  endif
 388
 389 #  define __inline_mathcode_(float_type, func, arg, code) \
 390   __MATH_INLINE float_type func (float_type) __THROW;                         \
 391   __inline_mathcodeNP_(float_type, func, arg, code)
 392
 393 #  define __inline_mathcodeNP_(float_type, func, arg, code) \
 394   __MATH_INLINE float_type __NTH (func (float_type arg))                      \
 395   {                                                                           \
 396     code;                                                                     \
 397   }
 398
 399
 400 #  define __inline_mathcode2_(float_type, func, arg1, arg2, code) \
 401   __MATH_INLINE float_type func (float_type, float_type) __THROW;             \
 402   __inline_mathcodeNP2_ (float_type, func, arg1, arg2, code)
 403
 404 #  define __inline_mathcodeNP2_(float_type, func, arg1, arg2, code) \
 405   __MATH_INLINE float_type __NTH (func (float_type arg1, float_type arg2))    \
 406   {                                                                           \
 407     code;                                                                     \
 408   }
 409
 410 #  define __inline_mathcode3_(float_type, func, arg1, arg2, arg3, code) \
 411   __MATH_INLINE float_type func (float_type, float_type, float_type) __THROW; \
 412   __inline_mathcodeNP3_(float_type, func, arg1, arg2, arg3, code)
 413
 414 #  define __inline_mathcodeNP3_(float_type, func, arg1, arg2, arg3, code) \
 415   __MATH_INLINE float_type __NTH (func (float_type arg1, float_type arg2,     \
 416                                         float_type arg3))                     \
 417   {                                                                           \
 418     code;                                                                     \
 419   }
 420 # endif
 421
 422
 423 # if !defined __NO_MATH_INLINES && defined __OPTIMIZE__
 424 /* Miscellaneous functions  */
 425
 426 /* __FAST_MATH__ is defined by gcc -ffast-math.  */
 427 #  ifdef __FAST_MATH__
 428 #   ifdef __USE_GNU
 429 #    define __sincos_code \
 430   register long double __cosr;                                                \
 431   register long double __sinr;                                                \
 432   register unsigned int __swtmp;                                              \
 433   __asm __volatile__                                                          \
 434     ("fsincos\n\t"                                                            \
 435      "fnstsw    %w2\n\t"                                                      \
 436      "testl     $0x400, %2\n\t"                                               \
 437      "jz        1f\n\t"                                                       \
 438      "fldpi\n\t"                                                              \
 439      "fadd      %%st(0)\n\t"                                                  \
 440      "fxch      %%st(1)\n\t"                                                  \
 441      "2: fprem1\n\t"                                                          \
 442      "fnstsw    %w2\n\t"                                                      \
 443      "testl     $0x400, %2\n\t"                                               \
 444      "jnz       2b\n\t"                                                       \
 445      "fstp      %%st(1)\n\t"                                                  \
 446      "fsincos\n\t"                                                            \
 447      "1:"                                                                     \
 448      : "=t" (__cosr), "=u" (__sinr), "=a" (__swtmp) : "0" (__x));             \
 449   *__sinx = __sinr;                                                           \
 450   *__cosx = __cosr
 451
 452 __MATH_INLINE void
 453 __NTH (__sincos (double __x, double *__sinx, double *__cosx))
 454 {
 455   __sincos_code;
 456 }
 457
 458 __MATH_INLINE void
 459 __NTH (__sincosf (float __x, float *__sinx, float *__cosx))
 460 {
 461   __sincos_code;
 462 }
 463
 464 __MATH_INLINE void
 465 __NTH (__sincosl (long double __x, long double *__sinx, long double *__cosx))
 466 {
 467   __sincos_code;
 468 }
 469 #   endif
 470
 471
 472 /* Optimized inline implementation, sometimes with reduced precision
 473    and/or argument range.  */
 474
 475 #   if __GNUC_PREREQ (3, 5)
 476 #    define __expm1_code \
 477   register long double __temp;                                                \
 478   __temp = __builtin_expm1l (__x);                                            \
 479   return __temp ? __temp : __x
 480 #   else
 481 #    define __expm1_code \
 482   register long double __value;                                               \
 483   register long double __exponent;                                            \
 484   register long double __temp;                                                \
 485   __asm __volatile__                                                          \
 486     ("fldl2e                    # e^x - 1 = 2^(x * log2(e)) - 1\n\t"          \
 487      "fmul      %%st(1)         # x * log2(e)\n\t"                            \
 488      "fst       %%st(1)\n\t"                                                  \
 489      "frndint                   # int(x * log2(e))\n\t"                       \
 490      "fxch\n\t"                                                               \
 491      "fsub      %%st(1)         # fract(x * log2(e))\n\t"                     \
 492      "f2xm1                     # 2^(fract(x * log2(e))) - 1\n\t"             \
 493      "fscale                    # 2^(x * log2(e)) - 2^(int(x * log2(e)))\n\t" \
 494      : "=t" (__value), "=u" (__exponent) : "0" (__x));                        \
 495   __asm __volatile__                                                          \
 496     ("fscale                    # 2^int(x * log2(e))\n\t"                     \
 497      : "=t" (__temp) : "0" (1.0), "u" (__exponent));                          \
 498   __temp -= 1.0;                                                              \
 499   __temp += __value;                                                          \
 500   return __temp ? __temp : __x
 501 #   endif
 502 __inline_mathcodeNP_ (long double, __expm1l, __x, __expm1_code)
 503
 504 #   if __GNUC_PREREQ (3, 4)
 505 __inline_mathcodeNP_ (long double, __expl, __x, return __builtin_expl (__x))
 506 #   else
 507 #    define __exp_code \
 508   register long double __value;                                               \
 509   register long double __exponent;                                            \
 510   __asm __volatile__                                                          \
 511     ("fldl2e                    # e^x = 2^(x * log2(e))\n\t"                  \
 512      "fmul      %%st(1)         # x * log2(e)\n\t"                            \
 513      "fst       %%st(1)\n\t"                                                  \
 514      "frndint                   # int(x * log2(e))\n\t"                       \
 515      "fxch\n\t"                                                               \
 516      "fsub      %%st(1)         # fract(x * log2(e))\n\t"                     \
 517      "f2xm1                     # 2^(fract(x * log2(e))) - 1\n\t"             \
 518      : "=t" (__value), "=u" (__exponent) : "0" (__x));                        \
 519   __value += 1.0;                                                             \
 520   __asm __volatile__                                                          \
 521     ("fscale"                                                                 \
 522      : "=t" (__value) : "0" (__value), "u" (__exponent));                     \
 523   return __value
 524 __inline_mathcodeNP (exp, __x, __exp_code)
 525 __inline_mathcodeNP_ (long double, __expl, __x, __exp_code)
 526 #   endif
 527
 528
 529 #   if !__GNUC_PREREQ (3, 5)
 530 __inline_mathcodeNP (tan, __x, \
 531   register long double __value;                                               \
 532   register long double __value2 __attribute__ ((__unused__));                 \
 533   __asm __volatile__                                                          \
 534     ("fptan"                                                                  \
 535      : "=t" (__value2), "=u" (__value) : "0" (__x));                          \
 536   return __value)
 537 #   endif
 538 #  endif /* __FAST_MATH__ */
 539
 540
 541 #  if __GNUC_PREREQ (3, 4)
 542 __inline_mathcodeNP2_ (long double, __atan2l, __y, __x,
 543                        return __builtin_atan2l (__y, __x))
 544 #  else
 545 #   define __atan2_code \
 546   register long double __value;                                               \
 547   __asm __volatile__                                                          \
 548     ("fpatan"                                                                 \
 549      : "=t" (__value) : "0" (__x), "u" (__y) : "st(1)");                      \
 550   return __value
 551 #   ifdef __FAST_MATH__
 552 __inline_mathcodeNP2 (atan2, __y, __x, __atan2_code)
 553 #   endif
 554 __inline_mathcodeNP2_ (long double, __atan2l, __y, __x, __atan2_code)
 555 #  endif
 556
 557
 558 #  if defined __FAST_MATH__ && !__GNUC_PREREQ (3, 5)
 559 __inline_mathcodeNP2 (fmod, __x, __y, \
 560   register long double __value;                                               \
 561   __asm __volatile__                                                          \
 562     ("1:        fprem\n\t"                                                    \
 563      "fnstsw    %%ax\n\t"                                                     \
 564      "sahf\n\t"                                                               \
 565      "jp        1b"                                                           \
 566      : "=t" (__value) : "0" (__x), "u" (__y) : "ax", "cc");                   \
 567   return __value)
 568 #  endif
 569
 570
 571 #  ifdef __FAST_MATH__
 572 #   if !__GNUC_PREREQ (3,3)
 573 __inline_mathopNP (sqrt, "fsqrt")
 574 __inline_mathopNP_ (long double, __sqrtl, "fsqrt")
 575 #    define __libc_sqrtl(n) __sqrtl (n)
 576 #   else
 577 #    define __libc_sqrtl(n) __builtin_sqrtl (n)
 578 #   endif
 579 #  endif
 580
 581 #  if __GNUC_PREREQ (2, 8)
 582 __inline_mathcodeNP_ (double, fabs, __x, return __builtin_fabs (__x))
 583 #   ifdef __USE_ISOC99
 584 __inline_mathcodeNP_ (float, fabsf, __x, return __builtin_fabsf (__x))
 585 __inline_mathcodeNP_ (long double, fabsl, __x, return __builtin_fabsl (__x))
 586 #   endif
 587 __inline_mathcodeNP_ (long double, __fabsl, __x, return __builtin_fabsl (__x))
 588 #  else
 589 __inline_mathop (fabs, "fabs")
 590 __inline_mathop_ (long double, __fabsl, "fabs")
 591 # endif
 592
 593 #  ifdef __FAST_MATH__
 594 #   if !__GNUC_PREREQ (3, 4)
 595 /* The argument range of this inline version is reduced.  */
 596 __inline_mathopNP (sin, "fsin")
 597 /* The argument range of this inline version is reduced.  */
 598 __inline_mathopNP (cos, "fcos")
 599
 600 __inline_mathop_declNP (log, "fldln2; fxch; fyl2x", "0" (__x) : "st(1)")
 601 #   endif
 602
 603 #   if !__GNUC_PREREQ (3, 5)
 604 __inline_mathop_declNP (log10, "fldlg2; fxch; fyl2x", "0" (__x) : "st(1)")
 605
 606 __inline_mathcodeNP (asin, __x, return __atan2l (__x, __libc_sqrtl (1.0 - __x * __x)))
 607 __inline_mathcodeNP (acos, __x, return __atan2l (__libc_sqrtl (1.0 - __x * __x), __x))
 608 #   endif
 609
 610 #   if !__GNUC_PREREQ (3, 4)
 611 __inline_mathop_declNP (atan, "fld1; fpatan", "0" (__x) : "st(1)")
 612 #   endif
 613 #  endif /* __FAST_MATH__ */
 614
 615 __inline_mathcode_ (long double, __sgn1l, __x, \
 616   __extension__ union { long double __xld; unsigned int __xi[3]; } __n =      \
 617     { __xld: __x };                                                           \
 618   __n.__xi[2] = (__n.__xi[2] & 0x8000) | 0x3fff;                              \
 619   __n.__xi[1] = 0x80000000;                                                   \
 620   __n.__xi[0] = 0;                                                            \
 621   return __n.__xld)
 622
 623
 624 #  ifdef __FAST_MATH__
 625 /* The argument range of the inline version of sinhl is slightly reduced.  */
 626 __inline_mathcodeNP (sinh, __x, \
 627   register long double __exm1 = __expm1l (__fabsl (__x));                     \
 628   return 0.5 * (__exm1 / (__exm1 + 1.0) + __exm1) * __sgn1l (__x))
 629
 630 __inline_mathcodeNP (cosh, __x, \
 631   register long double __ex = __expl (__x);                                   \
 632   return 0.5 * (__ex + 1.0 / __ex))
 633
 634 __inline_mathcodeNP (tanh, __x, \
 635   register long double __exm1 = __expm1l (-__fabsl (__x + __x));              \
 636   return __exm1 / (__exm1 + 2.0) * __sgn1l (-__x))
 637 #  endif
 638
 639 __inline_mathcodeNP (floor, __x, \
 640   register long double __value;                                               \
 641   register int __ignore;                                                      \
 642   unsigned short int __cw;                                                    \
 643   unsigned short int __cwtmp;                                                 \
 644   __asm __volatile ("fnstcw %3\n\t"                                           \
 645                     "movzwl %3, %1\n\t"                                       \
 646                     "andl $0xf3ff, %1\n\t"                                    \
 647                     "orl $0x0400, %1\n\t"       /* rounding down */           \
 648                     "movw %w1, %2\n\t"                                        \
 649                     "fldcw %2\n\t"                                            \
 650                     "frndint\n\t"                                             \
 651                     "fldcw %3"                                                \
 652                     : "=t" (__value), "=&q" (__ignore), "=m" (__cwtmp),       \
 653                       "=m" (__cw)                                             \
 654                     : "0" (__x));                                             \
 655   return __value)
 656
 657 __inline_mathcodeNP (ceil, __x, \
 658   register long double __value;                                               \
 659   register int __ignore;                                                      \
 660   unsigned short int __cw;                                                    \
 661   unsigned short int __cwtmp;                                                 \
 662   __asm __volatile ("fnstcw %3\n\t"                                           \
 663                     "movzwl %3, %1\n\t"                                       \
 664                     "andl $0xf3ff, %1\n\t"                                    \
 665                     "orl $0x0800, %1\n\t"       /* rounding up */             \
 666                     "movw %w1, %2\n\t"                                        \
 667                     "fldcw %2\n\t"                                            \
 668                     "frndint\n\t"                                             \
 669                     "fldcw %3"                                                \
 670                     : "=t" (__value), "=&q" (__ignore), "=m" (__cwtmp),       \
 671                       "=m" (__cw)                                             \
 672                     : "0" (__x));                                             \
 673   return __value)
 674
 675 #  ifdef __FAST_MATH__
 676 #   define __ldexp_code \
 677   register long double __value;                                               \
 678   __asm __volatile__                                                          \
 679     ("fscale"                                                                 \
 680      : "=t" (__value) : "0" (__x), "u" ((long double) __y));                  \
 681   return __value
 682
 683 __MATH_INLINE double
 684 __NTH (ldexp (double __x, int __y))
 685 {
 686   __ldexp_code;
 687 }
 688 #  endif
 689
 690
 691 /* Optimized versions for some non-standardized functions.  */
 692 #  ifdef __USE_ISOC99
 693
 694 #   ifdef __FAST_MATH__
 695 __inline_mathcodeNP (expm1, __x, __expm1_code)
 696
 697 /* We cannot rely on M_SQRT being defined.  So we do it for ourself
 698    here.  */
 699 #    define __M_SQRT2   1.41421356237309504880L /* sqrt(2) */
 700
 701 #    if !__GNUC_PREREQ (3, 5)
 702 __inline_mathcodeNP (log1p, __x, \
 703   register long double __value;                                               \
 704   if (__fabsl (__x) >= 1.0 - 0.5 * __M_SQRT2)                                 \
 705     __value = logl (1.0 + __x);                                               \
 706   else                                                                        \
 707     __asm __volatile__                                                        \
 708       ("fldln2\n\t"                                                           \
 709        "fxch\n\t"                                                             \
 710        "fyl2xp1"                                                              \
 711        : "=t" (__value) : "0" (__x) : "st(1)");                               \
 712   return __value)
 713 #    endif
 714
 715
 716 /* The argument range of the inline version of asinhl is slightly reduced.  */
 717 __inline_mathcodeNP (asinh, __x, \
 718   register long double  __y = __fabsl (__x);                                  \
 719   return (log1pl (__y * __y / (__libc_sqrtl (__y * __y + 1.0) + 1.0) + __y)   \
 720           * __sgn1l (__x)))
 721
 722 __inline_mathcodeNP (acosh, __x, \
 723   return logl (__x + __libc_sqrtl (__x - 1.0) * __libc_sqrtl (__x + 1.0)))
 724
 725 __inline_mathcodeNP (atanh, __x, \
 726   register long double __y = __fabsl (__x);                                   \
 727   return -0.5 * log1pl (-(__y + __y) / (1.0 + __y)) * __sgn1l (__x))
 728
 729 /* The argument range of the inline version of hypotl is slightly reduced.  */
 730 __inline_mathcodeNP2 (hypot, __x, __y,
 731                       return __libc_sqrtl (__x * __x + __y * __y))
 732
 733 #    if !__GNUC_PREREQ (3, 5)
 734 __inline_mathcodeNP(logb, __x, \
 735   register long double __value;                                               \
 736   register long double __junk;                                                \
 737   __asm __volatile__                                                          \
 738     ("fxtract\n\t"                                                            \
 739      : "=t" (__junk), "=u" (__value) : "0" (__x));                            \
 740   return __value)
 741 #    endif
 742
 743 #   endif
 744 #  endif
 745
 746 #  ifdef __USE_ISOC99
 747 #   ifdef __FAST_MATH__
 748
 749 #    if !__GNUC_PREREQ (3, 5)
 750 __inline_mathop_declNP (log2, "fld1; fxch; fyl2x", "0" (__x) : "st(1)")
 751 #    endif
 752
 753 __MATH_INLINE float
 754 __NTH (ldexpf (float __x, int __y))
 755 {
 756   __ldexp_code;
 757 }
 758
 759 __MATH_INLINE long double
 760 __NTH (ldexpl (long double __x, int __y))
 761 {
 762   __ldexp_code;
 763 }
 764
 765 __inline_mathopNP (rint, "frndint")
 766 #   endif /* __FAST_MATH__ */
 767
 768 #   define __lrint_code \
 769   long int __lrintres;                                                        \
 770   __asm__ __volatile__                                                        \
 771     ("fistpl %0"                                                              \
 772      : "=m" (__lrintres) : "t" (__x) : "st");                                 \
 773   return __lrintres
 774 __MATH_INLINE long int
 775 __NTH (lrintf (float __x))
 776 {
 777   __lrint_code;
 778 }
 779 __MATH_INLINE long int
 780 __NTH (lrint (double __x))
 781 {
 782   __lrint_code;
 783 }
 784 __MATH_INLINE long int
 785 __NTH (lrintl (long double __x))
 786 {
 787   __lrint_code;
 788 }
 789 #   undef __lrint_code
 790
 791 #   define __llrint_code \
 792   long long int __llrintres;                                                  \
 793   __asm__ __volatile__                                                        \
 794     ("fistpll %0"                                                             \
 795      : "=m" (__llrintres) : "t" (__x) : "st");                                \
 796   return __llrintres
 797 __extension__
 798 __MATH_INLINE long long int
 799 __NTH (llrintf (float __x))
 800 {
 801   __llrint_code;
 802 }
 803 __extension__
 804 __MATH_INLINE long long int
 805 __NTH (llrint (double __x))
 806 {
 807   __llrint_code;
 808 }
 809 __extension__
 810 __MATH_INLINE long long int
 811 __NTH (llrintl (long double __x))
 812 {
 813   __llrint_code;
 814 }
 815 #   undef __llrint_code
 816
 817 # endif
 818
 819
 820 #  ifdef __USE_MISC
 821
 822 #   if defined __FAST_MATH__ && !__GNUC_PREREQ (3, 5)
 823 __inline_mathcodeNP2 (drem, __x, __y, \
 824   register double __value;                                                    \
 825   register int __clobbered;                                                   \
 826   __asm __volatile__                                                          \
 827     ("1:        fprem1\n\t"                                                   \
 828      "fstsw     %%ax\n\t"                                                     \
 829      "sahf\n\t"                                                               \
 830      "jp        1b"                                                           \
 831      : "=t" (__value), "=&a" (__clobbered) : "0" (__x), "u" (__y) : "cc");    \
 832   return __value)
 833 #  endif
 834
 835
 836 /* This function is used in the `isfinite' macro.  */
 837 __MATH_INLINE int
 838 __NTH (__finite (double __x))
 839 {
 840   return (__extension__
 841           (((((union { double __d; int __i[2]; }) {__d: __x}).__i[1]
 842              | 0x800fffffu) + 1) >> 31));
 843 }
 844
 845 #  endif /* __USE_MISC  */
 846
 847 /* Undefine some of the large macros which are not used anymore.  */
 848 #  undef __atan2_code
 849 #  ifdef __FAST_MATH__
 850 #   undef __expm1_code
 851 #   undef __exp_code
 852 #   undef __sincos_code
 853 #  endif /* __FAST_MATH__ */
 854
 855 # endif /* __NO_MATH_INLINES  */
 856
 857
 858 /* This code is used internally in the GNU libc.  */
 859 # ifdef __LIBC_INTERNAL_MATH_INLINES
 860 __inline_mathop (__ieee754_sqrt, "fsqrt")
 861 __inline_mathcode2_ (long double, __ieee754_atan2l, __y, __x,
 862                      register long double __value;
 863                      __asm __volatile__ ("fpatan\n\t"
 864                                          : "=t" (__value)
 865                                          : "0" (__x), "u" (__y) : "st(1)");
 866                      return __value;)
 867 # endif
 868
 869 #endif /* !__SSE2_MATH__ && !__x86_64__ */