x86_64: Prefer EVEX512 code-path on AMD Zen5 CPUs

Introduced a synthetic architecture preference flag (Prefer_EVEX512)
and enabled it for AMD Zen5 (CPUID Family 0x1A) when AVX-512 is supported.

This flag modifies IFUNC dispatch to prefer 512-bit EVEX variants over
256-bit EVEX variants for string and memory functions on Zen5 processors,
leveraging their native 512-bit execution units for improved throughput.
When Prefer_EVEX512 is set, the dispatcher selects evex512 implementations;
otherwise, it falls back to evex (256-bit) variants.

The implementation updates the IFUNC selection logic in ifunc-avx2.h and
ifunc-evex.h to check for the Prefer_EVEX512 flag before dispatching to
EVEX512 implementations. This change affects six string/memory functions:

  - strchr
  - strlen
  - strnlen
  - strrchr
  - strchrnul
  - memchr

Benchmarks conducted on AMD Zen5 hardware demonstrate significant
performance improvements across all affected functions:

Function    Baseline   Patched    Avg         Avg        Avg      Max
            Variant    Variant    Baseline    Patched    Change   Improve
                                  (ns)        (ns)       %        %
------------+----------+----------+-----------+----------+--------+--------
STRCHR      evex       evex512    16.408      12.293     25.08%   37.69%
STRLEN      evex       evex512    16.862      11.436     32.18%   56.74%
STRNLEN     evex       evex512    18.493      11.762     36.40%   64.40%
STRRCHR     evex       evex512    15.154      10.874     28.24%   44.38%
STRCHRNUL   evex       evex512    16.464      12.605     23.44%   45.56%
MEMCHR      evex       evex512    9.984       8.268      17.19%   39.99%

Additionally, a tunable option (glibc.cpu.x86_cpu_features.preferred)
is provided to allow runtime control of the Prefer_EVEX512 flag for testing
and compatibility.

Reviewed-by: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian@amd.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index ccfef5565241be1565be657ebd2273bb1c92753f..73ee4f2bb83342964f7551246c3d747b6fa1cab8 100644 (file)
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -1009,6 +1009,12 @@ disable_tsx:
       cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
          &= ~bit_arch_Avoid_Non_Temporal_Memset;
 
+      /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
+         when AVX-512 is usable. */
+      if (family == 0x1A && CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
+          cpu_features->preferred[index_arch_Prefer_EVEX512]
+            |= bit_arch_Prefer_EVEX512;
+
       if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
        {
          /* Since the FMA4 bit is in CPUID_INDEX_80000001 and

diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c
index 51769a54937906eeb807bff0538dd70ad2255676..59dde5f1d7bcb59ab886f583c71c279fc773eebd 100644 (file)
--- a/sysdeps/x86/cpu-tunables.c
+++ b/sysdeps/x86/cpu-tunables.c
@@ -203,6 +203,12 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
                                                     11);
            }
          break;
+       case 14:
+           {
+             CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
+               (n, cpu_features, Prefer_EVEX512, AVX512F, 14);
+           }
+         break;
        case 15:
            {
              CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,

diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
index 8cab2ae248312053187a224aff01a10659bfb20d..74acb3fde1e2e758c8e70e193795a185ae3b48ae 100644 (file)
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
@@ -35,3 +35,4 @@ BIT (Prefer_FSRM)
 BIT (Avoid_Short_Distance_REP_MOVSB)
 BIT (Avoid_Non_Temporal_Memset)
 BIT (Avoid_STOSB)
+BIT (Prefer_EVEX512)

diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h
index bc7178d97ece09ed4f6748b2f35099df0f4ee38c..474466ba9366cf30b6bd0b928d01e4509e492a2e 100644 (file)
--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h
+++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h
@@ -1,4 +1,4 @@
-/* Common definition for ifunc selections optimized with SSE2 and AVX2.
+/* Common definition for ifunc selections optimized with SSE2, AVX2 and EVEX512.
    All versions must be listed in ifunc-impl-list.c.
    Copyright (C) 2017-2026 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
@@ -25,6 +25,10 @@
 
 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
 
+#ifdef USE_EVEX512
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
+#endif
+
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
 
@@ -44,8 +48,13 @@ IFUNC_SELECTOR (void)
     {
       if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
          && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
-       return OPTIMIZE (evex);
-
+      {
+#ifdef USE_EVEX512
+        if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+      return OPTIMIZE (evex512);
+#endif
+         return OPTIMIZE (evex);
+      }
       if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
        return OPTIMIZE (avx2_rtm);
 

diff --git a/sysdeps/x86_64/multiarch/ifunc-evex.h b/sysdeps/x86_64/multiarch/ifunc-evex.h
index 973a5b3d153206e6b09a92d3a080ddb6c3df9254..02ca749a7226ca438aec5bd6e1352ac34eb6e2d7 100644 (file)
--- a/sysdeps/x86_64/multiarch/ifunc-evex.h
+++ b/sysdeps/x86_64/multiarch/ifunc-evex.h
@@ -1,4 +1,4 @@
-/* Common definition for ifunc selection optimized with EVEX.
+/* Common definition for ifunc selection optimized with EVEX and EVEX512.
    All versions must be listed in ifunc-impl-list.c.
    Copyright (C) 2017-2026 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
@@ -22,6 +22,10 @@
 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden;
 
+#ifdef USE_EVEX512
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
+#endif
+
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
 
@@ -42,6 +46,11 @@ IFUNC_SELECTOR (void)
       if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
          && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
        {
+#ifdef USE_EVEX512
+      if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+        return OPTIMIZE (evex512);
+#endif
+
          if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
            return OPTIMIZE (evex_rtm);
 

diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c
index cea051878733a4985655bc17658025528e3c1718..b2a6c666a992ee6e22c0568beb4d22a280ec8e0a 100644 (file)
--- a/sysdeps/x86_64/multiarch/memchr.c
+++ b/sysdeps/x86_64/multiarch/memchr.c
@@ -24,6 +24,7 @@
 # undef memchr
 
 # define SYMBOL_NAME memchr
+# define USE_EVEX512 1
 # include "ifunc-evex.h"
 
 libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ());

diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c
index 1064bc8e5bcc4bccd22bcb34ab8109e8e1ca2ba5..f6bd36ba736daa5840ef48137075ffb92554a280 100644 (file)
--- a/sysdeps/x86_64/multiarch/strchr.c
+++ b/sysdeps/x86_64/multiarch/strchr.c
@@ -27,6 +27,7 @@
 # include <init-arch.h>
 
 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
 
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
@@ -46,7 +47,12 @@ IFUNC_SELECTOR (void)
     {
       if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
          && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
-       return OPTIMIZE (evex);
+       {
+         if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+           return OPTIMIZE (evex512);
+
+         return OPTIMIZE (evex);
+       }
 
       if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
        return OPTIMIZE (avx2_rtm);

diff --git a/sysdeps/x86_64/multiarch/strchrnul.c b/sysdeps/x86_64/multiarch/strchrnul.c
index f4bfc28c7b422db8ca3300e5e8caf12da3283db2..3bb93bb8124de2e41cc4b9656ddf78ff1bccce6f 100644 (file)
--- a/sysdeps/x86_64/multiarch/strchrnul.c
+++ b/sysdeps/x86_64/multiarch/strchrnul.c
@@ -26,6 +26,7 @@
 # undef strchrnul
 
 # define SYMBOL_NAME strchrnul
+# define USE_EVEX512 1
 # include "ifunc-avx2.h"
 
 libc_ifunc_redirected (__redirect_strchrnul, __strchrnul,

diff --git a/sysdeps/x86_64/multiarch/strlen.c b/sysdeps/x86_64/multiarch/strlen.c
index 95aa48e5d3141b5f69fd898c7609500d4efe334b..2f93abf1c898b76487b6369107e5be4e670adddf 100644 (file)
--- a/sysdeps/x86_64/multiarch/strlen.c
+++ b/sysdeps/x86_64/multiarch/strlen.c
@@ -24,6 +24,7 @@
 # undef strlen
 
 # define SYMBOL_NAME strlen
+# define USE_EVEX512 1
 # include "ifunc-avx2.h"
 
 libc_ifunc_redirected (__redirect_strlen, strlen, IFUNC_SELECTOR ());

diff --git a/sysdeps/x86_64/multiarch/strnlen.c b/sysdeps/x86_64/multiarch/strnlen.c
index bfe4d7abf0893e27c495cb6251f8b9305da1c70d..62e50a27e4467979703fa2266431b9f73d798b00 100644 (file)
--- a/sysdeps/x86_64/multiarch/strnlen.c
+++ b/sysdeps/x86_64/multiarch/strnlen.c
@@ -26,6 +26,7 @@
 # undef strnlen
 
 # define SYMBOL_NAME strnlen
+# define USE_EVEX512 1
 # include "ifunc-avx2.h"
 
 libc_ifunc_redirected (__redirect_strnlen, __strnlen, IFUNC_SELECTOR ());

diff --git a/sysdeps/x86_64/multiarch/strrchr.c b/sysdeps/x86_64/multiarch/strrchr.c
index 66d3f5b1d1388f3be3e7d803d79f89ce300d3a1b..0965938561fc1ade4e4c7c8390b3ceb7f90d4d29 100644 (file)
--- a/sysdeps/x86_64/multiarch/strrchr.c
+++ b/sysdeps/x86_64/multiarch/strrchr.c
@@ -23,6 +23,7 @@
 # undef strrchr
 
 # define SYMBOL_NAME strrchr
+# define USE_EVEX512 1
 # include "ifunc-avx2.h"
 
 libc_ifunc_redirected (__redirect_strrchr, strrchr, IFUNC_SELECTOR ());