Redirect fma calls to __fma in libm

include/math.h has a mechanism to redirect internal calls to various
libm functions, that can often be inlined by the compiler, to call
non-exported __* names for those functions in the case when the calls
aren't inlined, with the redirection being disabled when
NO_MATH_REDIRECT.  Add fma to the functions to which this mechanism is
applied.

At present, libm-internal fma calls (generally to __builtin_fma*
functions) are only done when it's known the call will be inlined,
with alternative code not relying on an fma operation being used in
the caller otherwise.  This patch is in preparation for adding the TS
18661 / C2X narrowing fma functions to glibc; it will be natural for
the narrowing function implementations to call the underlying fma
functions unconditionally, with this either being inlined or resulting
in an __fma* call.  (Using two levels of round-to-odd computation like
that, in the case where there isn't an fma hardware instruction, isn't
optimal but is certainly a lot simpler for the initial implementation
than writing different narrowing fma implementations for all the
various pairs of formats.)

Tested with build-many-glibcs.py that installed stripped shared
libraries are unchanged by the patch (using
<https://sourceware.org/pipermail/libc-alpha/2021-September/130991.html>
to fix installed library stripping in build-many-glibcs.py).  Also
tested for x86_64.

diff --git a/include/math.h b/include/math.h
index b4772d3d3b70b88a6f4bc5b9f4c1a5d229662347..fa11a710a6c152a4a27e02315ef455bddf8eedc5 100644 (file)
--- a/include/math.h
+++ b/include/math.h
@@ -155,6 +155,7 @@ fabsf128 (_Float128 x)
 #   endif
 #   define MATH_REDIRECT_UNARY_ARGS(TYPE) TYPE
 #   define MATH_REDIRECT_BINARY_ARGS(TYPE) TYPE, TYPE
+#   define MATH_REDIRECT_TERNARY_ARGS(TYPE) TYPE, TYPE, TYPE
 MATH_REDIRECT (sqrt, "__ieee754_", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (ceil, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (floor, "__", MATH_REDIRECT_UNARY_ARGS)
@@ -163,6 +164,7 @@ MATH_REDIRECT (rint, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (trunc, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (round, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (copysign, "__", MATH_REDIRECT_BINARY_ARGS)
+MATH_REDIRECT (fma, "__", MATH_REDIRECT_TERNARY_ARGS)
 #  endif
 # endif
 

diff --git a/math/s_fma.c b/math/s_fma.c
index 5b0afde6b846200ee91108075aed30540d911786..2dc5c5d2cb098cd3d83842f0e8edb4714f45a658 100644 (file)
--- a/math/s_fma.c
+++ b/math/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-double.h>
 

diff --git a/math/s_fmaf.c b/math/s_fmaf.c
index 401f0fc5aef56038b70e03e1f2ef1216bbbdeec5..f1ba0a0c49c231a1b17f57173feb7403a2121e8a 100644 (file)
--- a/math/s_fmaf.c
+++ b/math/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-float.h>
 

diff --git a/math/s_fmal.c b/math/s_fmal.c
index 6b13ea1d5fb5703a09526c108887df1cc2043915..47a68ed235fc0ca6fa46f064757a1dcb881803df 100644 (file)
--- a/math/s_fmal.c
+++ b/math/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-ldouble.h>
 

diff --git a/sysdeps/i386/i686/multiarch/s_fma.c b/sysdeps/i386/i686/multiarch/s_fma.c
index 7d66aaef7a5c8713b3e54f46213d394d139c1488..229f6c6526e422fd2ee0f3e31f7376473dfd074f 100644 (file)
--- a/sysdeps/i386/i686/multiarch/s_fma.c
+++ b/sysdeps/i386/i686/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>

diff --git a/sysdeps/i386/i686/multiarch/s_fmaf.c b/sysdeps/i386/i686/multiarch/s_fmaf.c
index b8394d05d0c055bdd49d17acb99e4f8154b300a2..97123235f9b010431eb33ca14f488db3f97f27dc 100644 (file)
--- a/sysdeps/i386/i686/multiarch/s_fmaf.c
+++ b/sysdeps/i386/i686/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>

diff --git a/sysdeps/ieee754/dbl-64/s_fma.c b/sysdeps/ieee754/dbl-64/s_fma.c
index 55ccf46c9a239541a42b02ede64f550ccc8c3df4..aa4336387a8d6f59b8a5abebf897b52c362c2650 100644 (file)
--- a/sysdeps/ieee754/dbl-64/s_fma.c
+++ b/sysdeps/ieee754/dbl-64/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>

diff --git a/sysdeps/ieee754/dbl-64/s_fmaf.c b/sysdeps/ieee754/dbl-64/s_fmaf.c
index 2b4da8b763874e588cb07dff96fe1fe35675c374..2eb9a50d94b58397b20d187c2105e9a193ab5e77 100644 (file)
--- a/sysdeps/ieee754/dbl-64/s_fmaf.c
+++ b/sysdeps/ieee754/dbl-64/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>

diff --git a/sysdeps/ieee754/float128/s_fmaf128.c b/sysdeps/ieee754/float128/s_fmaf128.c
index 6497895c8d5075dc707e97d541ec6abb948e0ab2..a900af6e3c76538d4ba053c11918bf55b0f4873f 100644 (file)
--- a/sysdeps/ieee754/float128/s_fmaf128.c
+++ b/sysdeps/ieee754/float128/s_fmaf128.c
@@ -1,2 +1,3 @@
+#define NO_MATH_REDIRECT
 #include <float128_private.h>
 #include "../ldbl-128/s_fmal.c"

diff --git a/sysdeps/ieee754/ldbl-128/s_fma.c b/sysdeps/ieee754/ldbl-128/s_fma.c
index 47673da3d789de6478fcf0eb985e99926e85c631..4795e717e8ac0b6d9c1ea52ebacb5bd97e7ae5f7 100644 (file)
--- a/sysdeps/ieee754/ldbl-128/s_fma.c
+++ b/sysdeps/ieee754/ldbl-128/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>

diff --git a/sysdeps/ieee754/ldbl-128/s_fmal.c b/sysdeps/ieee754/ldbl-128/s_fmal.c
index 0f5f5f1d74f4364300d4f292e5f6ba031b72f93b..aff9efca8c3577334a73183228f9b17611cb9615 100644 (file)
--- a/sysdeps/ieee754/ldbl-128/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>

diff --git a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
index f1b36c46425652a8f68f96adc3a4f50193b56caf..a989f4cb2052dc92278e0f6734b4ce22d502a4c1 100644 (file)
--- a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <fenv.h>
 #include <float.h>
 #include <math.h>

diff --git a/sysdeps/ieee754/ldbl-96/s_fma.c b/sysdeps/ieee754/ldbl-96/s_fma.c
index 8ad7e4dc64b938dbc29c2692e4bbb465f35c6383..417c27e53492f674ed5d2cea430da6c0a33a822e 100644 (file)
--- a/sysdeps/ieee754/ldbl-96/s_fma.c
+++ b/sysdeps/ieee754/ldbl-96/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>

diff --git a/sysdeps/ieee754/ldbl-96/s_fmal.c b/sysdeps/ieee754/ldbl-96/s_fmal.c
index fad8450b29f57c7405e6c628e001b7d1d61a4126..cd83df451b40f466a980f53476aea631b55ade2f 100644 (file)
--- a/sysdeps/ieee754/ldbl-96/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-96/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>

diff --git a/sysdeps/ieee754/soft-fp/s_fma.c b/sysdeps/ieee754/soft-fp/s_fma.c
index 9fd8ddb68359a939f1d248189cda70947f3e5204..4b0d6b50aab83440206d01073c4c6cbb8d7bdbba 100644 (file)
--- a/sysdeps/ieee754/soft-fp/s_fma.c
+++ b/sysdeps/ieee754/soft-fp/s_fma.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-double.h>

diff --git a/sysdeps/ieee754/soft-fp/s_fmaf.c b/sysdeps/ieee754/soft-fp/s_fmaf.c
index 2abb8c7f59f0dcf9f2cfb31f92c656aad19689bf..bed5990f06204f95926f6e69390e72b5c5b130d0 100644 (file)
--- a/sysdeps/ieee754/soft-fp/s_fmaf.c
+++ b/sysdeps/ieee754/soft-fp/s_fmaf.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-float.h>

diff --git a/sysdeps/ieee754/soft-fp/s_fmal.c b/sysdeps/ieee754/soft-fp/s_fmal.c
index 0a364f46528c6198ab78cd1a572b9751b694af95..aecec139235914aed80c69f71bd9e9f8cfd4db6d 100644 (file)
--- a/sysdeps/ieee754/soft-fp/s_fmal.c
+++ b/sysdeps/ieee754/soft-fp/s_fmal.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-ldouble.h>

diff --git a/sysdeps/riscv/rvd/s_fma.c b/sysdeps/riscv/rvd/s_fma.c
index 70c0cbc0727101573c4de9e7857f5be4b5dae285..7f39cefc711d0c438589986e03169a3f38b30a69 100644 (file)
--- a/sysdeps/riscv/rvd/s_fma.c
+++ b/sysdeps/riscv/rvd/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>

diff --git a/sysdeps/riscv/rvf/s_fmaf.c b/sysdeps/riscv/rvf/s_fmaf.c
index cb972fbf2fda5ac791fe48d68dcc294bbb419b87..6057a009db268716ca39b238fda5d426e5c11012 100644 (file)
--- a/sysdeps/riscv/rvf/s_fmaf.c
+++ b/sysdeps/riscv/rvf/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>

diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
index 804272ff8ea204d963f1c1d19907f4595199f8f4..95f2cea66cb3d8d2e5562f308b4098f856694b84 100644 (file)
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <math_ldbl_opt.h>

diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
index e82c4a4031d6853de62343a26bdddca6344bf3a9..cc7caa7cd283e7d3e312164daa1e5fec5d4042a4 100644 (file)
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>

diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
index 34b4f45150be25db24a5fbda7e253963027454d0..44066eed46472b86a7d25db65eb40e8982e2d5bf 100644 (file)
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-double.h>

diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
index 08f71c8bbab8fb6daeeee0c8950dc0a981e6dc83..3d24342454399325cf30e5ce3f5e4141a614e068 100644 (file)
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>

diff --git a/sysdeps/x86_64/fpu/multiarch/s_fma.c b/sysdeps/x86_64/fpu/multiarch/s_fma.c
index 8ebf72103f7927477b4365c34acfed54ee4999b3..89389dd77f490e13f0499efc9e7f3be22859103c 100644 (file)
--- a/sysdeps/x86_64/fpu/multiarch/s_fma.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>

diff --git a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
index c6738265632b242821639e9308ba94041a3084a2..8c1934685d830ca3adcd020f7c7135d856335513 100644 (file)
--- a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>