`/* Implementation of the MATMUL intrinsic
- Copyright (C) 2002-2017 Free Software Foundation, Inc.
+ Copyright (C) 2002-2024 Free Software Foundation, Inc.
Contributed by Paul Brook <paul@nowt.org>
This file is part of the GNU Fortran runtime library (libgfortran).
int blas_limit, blas_call gemm);
export_proto(matmul_'rtype_code`);
-'ifelse(rtype_letter,`r',dnl
-`#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2. Only generate
- an AVX2 function if we are dealing with integer. */
-#undef HAVE_AVX2
-#endif')
-`
-
/* Put exhaustive list of possible architectures here here, ORed together. */
#if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
`static void
'matmul_name` ('rtype` * const restrict retarray,
'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
- int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+ int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
static' include(matmul_internal.m4)dnl
`#endif /* HAVE_AVX2 */
static' include(matmul_internal.m4)dnl
`#endif /* HAVE_AVX512F */
+/* AMD-specifix funtions with AVX128 and FMA3/FMA4. */
+
+#if defined(HAVE_AVX) && defined(HAVE_FMA3) && defined(HAVE_AVX128)
+'define(`matmul_name',`matmul_'rtype_code`_avx128_fma3')dnl
+`void
+'matmul_name` ('rtype` * const restrict retarray,
+ 'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+ int blas_limit, blas_call gemm) __attribute__((__target__("avx,fma")));
+internal_proto('matmul_name`);
+#endif
+
+#if defined(HAVE_AVX) && defined(HAVE_FMA4) && defined(HAVE_AVX128)
+'define(`matmul_name',`matmul_'rtype_code`_avx128_fma4')dnl
+`void
+'matmul_name` ('rtype` * const restrict retarray,
+ 'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+ int blas_limit, blas_call gemm) __attribute__((__target__("avx,fma4")));
+internal_proto('matmul_name`);
+#endif
+
/* Function to fall back to if there is no special processor-specific version. */
'define(`matmul_name',`matmul_'rtype_code`_vanilla')dnl
`static' include(matmul_internal.m4)dnl
/* Currently, this is i386 only. Adjust for other architectures. */
-#include <config/i386/cpuinfo.h>
void matmul_'rtype_code` ('rtype` * const restrict retarray,
'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
int blas_limit, blas_call gemm)
{
static void (*matmul_p) ('rtype` * const restrict retarray,
'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
- int blas_limit, blas_call gemm) = NULL;
+ int blas_limit, blas_call gemm);
+
+ void (*matmul_fn) ('rtype` * const restrict retarray,
+ 'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
+ int blas_limit, blas_call gemm);
- if (matmul_p == NULL)
+ matmul_fn = __atomic_load_n (&matmul_p, __ATOMIC_RELAXED);
+ if (matmul_fn == NULL)
{
- matmul_p = matmul_'rtype_code`_vanilla;
- if (__cpu_model.__cpu_vendor == VENDOR_INTEL)
+ matmul_fn = matmul_'rtype_code`_vanilla;
+ if (__builtin_cpu_is ("intel"))
{
/* Run down the available processors in order of preference. */
#ifdef HAVE_AVX512F
- if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+ if (__builtin_cpu_supports ("avx512f"))
{
- matmul_p = matmul_'rtype_code`_avx512f;
- goto tailcall;
+ matmul_fn = matmul_'rtype_code`_avx512f;
+ goto store;
}
#endif /* HAVE_AVX512F */
#ifdef HAVE_AVX2
- if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+ if (__builtin_cpu_supports ("avx2")
+ && __builtin_cpu_supports ("fma"))
{
- matmul_p = matmul_'rtype_code`_avx2;
- goto tailcall;
+ matmul_fn = matmul_'rtype_code`_avx2;
+ goto store;
}
#endif
#ifdef HAVE_AVX
- if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX))
+ if (__builtin_cpu_supports ("avx"))
{
- matmul_p = matmul_'rtype_code`_avx;
- goto tailcall;
+ matmul_fn = matmul_'rtype_code`_avx;
+ goto store;
}
#endif /* HAVE_AVX */
}
+ else if (__builtin_cpu_is ("amd"))
+ {
+#if defined(HAVE_AVX) && defined(HAVE_FMA3) && defined(HAVE_AVX128)
+ if (__builtin_cpu_supports ("avx")
+ && __builtin_cpu_supports ("fma"))
+ {
+ matmul_fn = matmul_'rtype_code`_avx128_fma3;
+ goto store;
+ }
+#endif
+#if defined(HAVE_AVX) && defined(HAVE_FMA4) && defined(HAVE_AVX128)
+ if (__builtin_cpu_supports ("avx")
+ && __builtin_cpu_supports ("fma4"))
+ {
+ matmul_fn = matmul_'rtype_code`_avx128_fma4;
+ goto store;
+ }
+#endif
+
+ }
+ store:
+ __atomic_store_n (&matmul_p, matmul_fn, __ATOMIC_RELAXED);
}
-tailcall:
- (*matmul_p) (retarray, a, b, try_blas, blas_limit, gemm);
+ (*matmul_fn) (retarray, a, b, try_blas, blas_limit, gemm);
}
#else /* Just the vanilla function. */