/* Implementation of the MATMUL intrinsic
- Copyright (C) 2002-2020 Free Software Foundation, Inc.
+ Copyright (C) 2002-2023 Free Software Foundation, Inc.
Contributed by Paul Brook <paul@nowt.org>
This file is part of the GNU Fortran runtime library (libgfortran).
}
}
- if (rxstride == 1 && axstride == 1 && bxstride == 1)
+ if (rxstride == 1 && axstride == 1 && bxstride == 1
+ && GFC_DESCRIPTOR_RANK (b) != 1)
{
/* This block of code implements a tuned matmul, derived from
Superscalar GEMM-based level 3 BLAS, Beta version 0.1
}
}
}
- else if (axstride < aystride)
- {
- for (y = 0; y < ycount; y++)
- for (x = 0; x < xcount; x++)
- dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
-
- for (y = 0; y < ycount; y++)
- for (n = 0; n < count; n++)
- for (x = 0; x < xcount; x++)
- /* dest[x,y] += a[x,n] * b[n,y] */
- dest[x*rxstride + y*rystride] +=
- abase[x*axstride + n*aystride] *
- bbase[n*bxstride + y*bystride];
- }
else if (GFC_DESCRIPTOR_RANK (a) == 1)
{
const GFC_INTEGER_4 *restrict bbase_y;
dest[y*rxstride] = s;
}
}
+ else if (axstride < aystride)
+ {
+ for (y = 0; y < ycount; y++)
+ for (x = 0; x < xcount; x++)
+ dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
+
+ for (y = 0; y < ycount; y++)
+ for (n = 0; n < count; n++)
+ for (x = 0; x < xcount; x++)
+ /* dest[x,y] += a[x,n] * b[n,y] */
+ dest[x*rxstride + y*rystride] +=
+ abase[x*axstride + n*aystride] *
+ bbase[n*bxstride + y*bystride];
+ }
else
{
const GFC_INTEGER_4 *restrict abase_x;
}
}
- if (rxstride == 1 && axstride == 1 && bxstride == 1)
+ if (rxstride == 1 && axstride == 1 && bxstride == 1
+ && GFC_DESCRIPTOR_RANK (b) != 1)
{
/* This block of code implements a tuned matmul, derived from
Superscalar GEMM-based level 3 BLAS, Beta version 0.1
}
}
}
- else if (axstride < aystride)
- {
- for (y = 0; y < ycount; y++)
- for (x = 0; x < xcount; x++)
- dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
-
- for (y = 0; y < ycount; y++)
- for (n = 0; n < count; n++)
- for (x = 0; x < xcount; x++)
- /* dest[x,y] += a[x,n] * b[n,y] */
- dest[x*rxstride + y*rystride] +=
- abase[x*axstride + n*aystride] *
- bbase[n*bxstride + y*bystride];
- }
else if (GFC_DESCRIPTOR_RANK (a) == 1)
{
const GFC_INTEGER_4 *restrict bbase_y;
dest[y*rxstride] = s;
}
}
+ else if (axstride < aystride)
+ {
+ for (y = 0; y < ycount; y++)
+ for (x = 0; x < xcount; x++)
+ dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
+
+ for (y = 0; y < ycount; y++)
+ for (n = 0; n < count; n++)
+ for (x = 0; x < xcount; x++)
+ /* dest[x,y] += a[x,n] * b[n,y] */
+ dest[x*rxstride + y*rystride] +=
+ abase[x*axstride + n*aystride] *
+ bbase[n*bxstride + y*bystride];
+ }
else
{
const GFC_INTEGER_4 *restrict abase_x;
}
}
- if (rxstride == 1 && axstride == 1 && bxstride == 1)
+ if (rxstride == 1 && axstride == 1 && bxstride == 1
+ && GFC_DESCRIPTOR_RANK (b) != 1)
{
/* This block of code implements a tuned matmul, derived from
Superscalar GEMM-based level 3 BLAS, Beta version 0.1
}
}
}
- else if (axstride < aystride)
- {
- for (y = 0; y < ycount; y++)
- for (x = 0; x < xcount; x++)
- dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
-
- for (y = 0; y < ycount; y++)
- for (n = 0; n < count; n++)
- for (x = 0; x < xcount; x++)
- /* dest[x,y] += a[x,n] * b[n,y] */
- dest[x*rxstride + y*rystride] +=
- abase[x*axstride + n*aystride] *
- bbase[n*bxstride + y*bystride];
- }
else if (GFC_DESCRIPTOR_RANK (a) == 1)
{
const GFC_INTEGER_4 *restrict bbase_y;
dest[y*rxstride] = s;
}
}
+ else if (axstride < aystride)
+ {
+ for (y = 0; y < ycount; y++)
+ for (x = 0; x < xcount; x++)
+ dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
+
+ for (y = 0; y < ycount; y++)
+ for (n = 0; n < count; n++)
+ for (x = 0; x < xcount; x++)
+ /* dest[x,y] += a[x,n] * b[n,y] */
+ dest[x*rxstride + y*rystride] +=
+ abase[x*axstride + n*aystride] *
+ bbase[n*bxstride + y*bystride];
+ }
else
{
const GFC_INTEGER_4 *restrict abase_x;
}
}
- if (rxstride == 1 && axstride == 1 && bxstride == 1)
+ if (rxstride == 1 && axstride == 1 && bxstride == 1
+ && GFC_DESCRIPTOR_RANK (b) != 1)
{
/* This block of code implements a tuned matmul, derived from
Superscalar GEMM-based level 3 BLAS, Beta version 0.1
}
}
}
- else if (axstride < aystride)
- {
- for (y = 0; y < ycount; y++)
- for (x = 0; x < xcount; x++)
- dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
-
- for (y = 0; y < ycount; y++)
- for (n = 0; n < count; n++)
- for (x = 0; x < xcount; x++)
- /* dest[x,y] += a[x,n] * b[n,y] */
- dest[x*rxstride + y*rystride] +=
- abase[x*axstride + n*aystride] *
- bbase[n*bxstride + y*bystride];
- }
else if (GFC_DESCRIPTOR_RANK (a) == 1)
{
const GFC_INTEGER_4 *restrict bbase_y;
dest[y*rxstride] = s;
}
}
+ else if (axstride < aystride)
+ {
+ for (y = 0; y < ycount; y++)
+ for (x = 0; x < xcount; x++)
+ dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
+
+ for (y = 0; y < ycount; y++)
+ for (n = 0; n < count; n++)
+ for (x = 0; x < xcount; x++)
+ /* dest[x,y] += a[x,n] * b[n,y] */
+ dest[x*rxstride + y*rystride] +=
+ abase[x*axstride + n*aystride] *
+ bbase[n*bxstride + y*bystride];
+ }
else
{
const GFC_INTEGER_4 *restrict abase_x;
/* Currently, this is i386 only. Adjust for other architectures. */
-#include <config/i386/cpuinfo.h>
void matmul_i4 (gfc_array_i4 * const restrict retarray,
gfc_array_i4 * const restrict a, gfc_array_i4 * const restrict b, int try_blas,
int blas_limit, blas_call gemm)
if (matmul_fn == NULL)
{
matmul_fn = matmul_i4_vanilla;
- if (__cpu_model.__cpu_vendor == VENDOR_INTEL)
+ if (__builtin_cpu_is ("intel"))
{
/* Run down the available processors in order of preference. */
#ifdef HAVE_AVX512F
- if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+ if (__builtin_cpu_supports ("avx512f"))
{
matmul_fn = matmul_i4_avx512f;
goto store;
#endif /* HAVE_AVX512F */
#ifdef HAVE_AVX2
- if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
- && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+ if (__builtin_cpu_supports ("avx2")
+ && __builtin_cpu_supports ("fma"))
{
matmul_fn = matmul_i4_avx2;
goto store;
#endif
#ifdef HAVE_AVX
- if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX))
+ if (__builtin_cpu_supports ("avx"))
{
matmul_fn = matmul_i4_avx;
goto store;
}
#endif /* HAVE_AVX */
}
- else if (__cpu_model.__cpu_vendor == VENDOR_AMD)
+ else if (__builtin_cpu_is ("amd"))
{
#if defined(HAVE_AVX) && defined(HAVE_FMA3) && defined(HAVE_AVX128)
- if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX))
- && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+ if (__builtin_cpu_supports ("avx")
+ && __builtin_cpu_supports ("fma"))
{
matmul_fn = matmul_i4_avx128_fma3;
goto store;
}
#endif
#if defined(HAVE_AVX) && defined(HAVE_FMA4) && defined(HAVE_AVX128)
- if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX))
- && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA4)))
+ if (__builtin_cpu_supports ("avx")
+ && __builtin_cpu_supports ("fma4"))
{
matmul_fn = matmul_i4_avx128_fma4;
goto store;
}
}
- if (rxstride == 1 && axstride == 1 && bxstride == 1)
+ if (rxstride == 1 && axstride == 1 && bxstride == 1
+ && GFC_DESCRIPTOR_RANK (b) != 1)
{
/* This block of code implements a tuned matmul, derived from
Superscalar GEMM-based level 3 BLAS, Beta version 0.1
}
}
}
- else if (axstride < aystride)
- {
- for (y = 0; y < ycount; y++)
- for (x = 0; x < xcount; x++)
- dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
-
- for (y = 0; y < ycount; y++)
- for (n = 0; n < count; n++)
- for (x = 0; x < xcount; x++)
- /* dest[x,y] += a[x,n] * b[n,y] */
- dest[x*rxstride + y*rystride] +=
- abase[x*axstride + n*aystride] *
- bbase[n*bxstride + y*bystride];
- }
else if (GFC_DESCRIPTOR_RANK (a) == 1)
{
const GFC_INTEGER_4 *restrict bbase_y;
dest[y*rxstride] = s;
}
}
+ else if (axstride < aystride)
+ {
+ for (y = 0; y < ycount; y++)
+ for (x = 0; x < xcount; x++)
+ dest[x*rxstride + y*rystride] = (GFC_INTEGER_4)0;
+
+ for (y = 0; y < ycount; y++)
+ for (n = 0; n < count; n++)
+ for (x = 0; x < xcount; x++)
+ /* dest[x,y] += a[x,n] * b[n,y] */
+ dest[x*rxstride + y*rystride] +=
+ abase[x*axstride + n*aystride] *
+ bbase[n*bxstride + y*bystride];
+ }
else
{
const GFC_INTEGER_4 *restrict abase_x;