Update copyright years.

[thirdparty/gcc.git] / libgfortran / m4 / matmul.m4

diff --git a/libgfortran/m4/matmul.m4 b/libgfortran/m4/matmul.m4
index 4e5bf60620905e9a92fc08ab06c49b151cb50ce6..7fc1f5fa75fb1ce4f4607e1d2e2e2310dfe37f28 100644 (file)
--- a/libgfortran/m4/matmul.m4
+++ b/libgfortran/m4/matmul.m4
@@ -1,5 +1,5 @@
 `/* Implementation of the MATMUL intrinsic
-   Copyright (C) 2002-2016 Free Software Foundation, Inc.
+   Copyright (C) 2002-2024 Free Software Foundation, Inc.
    Contributed by Paul Brook <paul@nowt.org>
 
 This file is part of the GNU Fortran runtime library (libgfortran).
@@ -24,7 +24,6 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 <http://www.gnu.org/licenses/>.  */
 
 #include "libgfortran.h"
-#include <stdlib.h>
 #include <string.h>
 #include <assert.h>'
 
@@ -76,14 +75,6 @@ extern void matmul_'rtype_code` ('rtype` * const restrict retarray,
        int blas_limit, blas_call gemm);
 export_proto(matmul_'rtype_code`);
 
-'ifelse(rtype_letter,`r',dnl
-`#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif')
-`
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -102,7 +93,7 @@ static' include(matmul_internal.m4)dnl
 `static void
 'matmul_name` ('rtype` * const restrict retarray, 
        'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
-       int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+       int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static' include(matmul_internal.m4)dnl
 `#endif /* HAVE_AVX2 */
 
@@ -115,6 +106,26 @@ static' include(matmul_internal.m4)dnl
 static' include(matmul_internal.m4)dnl
 `#endif  /* HAVE_AVX512F */
 
+/* AMD-specifix funtions with AVX128 and FMA3/FMA4.  */
+
+#if defined(HAVE_AVX) && defined(HAVE_FMA3) && defined(HAVE_AVX128)
+'define(`matmul_name',`matmul_'rtype_code`_avx128_fma3')dnl
+`void
+'matmul_name` ('rtype` * const restrict retarray, 
+       'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+       int blas_limit, blas_call gemm) __attribute__((__target__("avx,fma")));
+internal_proto('matmul_name`);
+#endif
+
+#if defined(HAVE_AVX) && defined(HAVE_FMA4) && defined(HAVE_AVX128)
+'define(`matmul_name',`matmul_'rtype_code`_avx128_fma4')dnl
+`void
+'matmul_name` ('rtype` * const restrict retarray, 
+       'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+       int blas_limit, blas_call gemm) __attribute__((__target__("avx,fma4")));
+internal_proto('matmul_name`);
+#endif
+
 /* Function to fall back to if there is no special processor-specific version.  */
 'define(`matmul_name',`matmul_'rtype_code`_vanilla')dnl
 `static' include(matmul_internal.m4)dnl
@@ -123,51 +134,77 @@ static' include(matmul_internal.m4)dnl
 
 /* Currently, this is i386 only.  Adjust for other architectures.  */
 
-#include <config/i386/cpuinfo.h>
 void matmul_'rtype_code` ('rtype` * const restrict retarray, 
        'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
        int blas_limit, blas_call gemm)
 {
   static void (*matmul_p) ('rtype` * const restrict retarray, 
        'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
-       int blas_limit, blas_call gemm) = NULL;
+       int blas_limit, blas_call gemm);
+
+  void (*matmul_fn) ('rtype` * const restrict retarray, 
+       'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+       int blas_limit, blas_call gemm);
 
-  if (matmul_p == NULL)
+  matmul_fn = __atomic_load_n (&matmul_p, __ATOMIC_RELAXED);
+  if (matmul_fn == NULL)
     {
-      matmul_p = matmul_'rtype_code`_vanilla;
-      if (__cpu_model.__cpu_vendor == VENDOR_INTEL)
+      matmul_fn = matmul_'rtype_code`_vanilla;
+      if (__builtin_cpu_is ("intel"))
        {
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-         if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+         if (__builtin_cpu_supports ("avx512f"))
            {
-             matmul_p = matmul_'rtype_code`_avx512f;
-             goto tailcall;
+             matmul_fn = matmul_'rtype_code`_avx512f;
+             goto store;
            }
 
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-         if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+         if (__builtin_cpu_supports ("avx2")
+             && __builtin_cpu_supports ("fma"))
            {
-             matmul_p = matmul_'rtype_code`_avx2;
-             goto tailcall;
+             matmul_fn = matmul_'rtype_code`_avx2;
+             goto store;
            }
 
 #endif
 
 #ifdef HAVE_AVX
-         if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX))
+         if (__builtin_cpu_supports ("avx"))
            {
-              matmul_p = matmul_'rtype_code`_avx;
-             goto tailcall;
+              matmul_fn = matmul_'rtype_code`_avx;
+             goto store;
            }
 #endif  /* HAVE_AVX */
         }
+    else if (__builtin_cpu_is ("amd"))
+      {
+#if defined(HAVE_AVX) && defined(HAVE_FMA3) && defined(HAVE_AVX128)
+       if (__builtin_cpu_supports ("avx")
+           && __builtin_cpu_supports ("fma"))
+         {
+            matmul_fn = matmul_'rtype_code`_avx128_fma3;
+           goto store;
+         }
+#endif
+#if defined(HAVE_AVX) && defined(HAVE_FMA4) && defined(HAVE_AVX128)
+       if (__builtin_cpu_supports ("avx")
+           && __builtin_cpu_supports ("fma4"))
+         {
+            matmul_fn = matmul_'rtype_code`_avx128_fma4;
+           goto store;
+         }
+#endif
+
+      }
+   store:
+      __atomic_store_n (&matmul_p, matmul_fn, __ATOMIC_RELAXED);
    }
 
-tailcall:
-   (*matmul_p) (retarray, a, b, try_blas, blas_limit, gemm);
+   (*matmul_fn) (retarray, a, b, try_blas, blas_limit, gemm);
 }
 
 #else  /* Just the vanilla function.  */