x86-64: Use ZMM16-ZMM31 in AVX512 memmove family functions

Update ifunc-memmove.h to select the function optimized with AVX512
instructions using ZMM16-ZMM31 registers to avoid RTM abort with usable
AVX512VL since VZEROUPPER isn't needed at function exit.

(cherry picked from commit e4fda4631017e49d4ee5a2755db34289b6860fa4)

diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 2016072489512bb1f4bef73a77d3151f22245cf5..b0ae7ed0dac08377d9a43382fe8cc870c398005d 100644 (file)
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -83,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __memmove_chk_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, __memmove_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memmove_chk_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, __memmove_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memmove_chk_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, __memmove_chk,
                              HAS_ARCH_FEATURE (AVX_Usable),
@@ -148,10 +148,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __memmove_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, memmove,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memmove_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, memmove,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memmove_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, memmove, HAS_CPU_FEATURE (SSSE3),
                              __memmove_ssse3_back)
@@ -733,10 +733,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __memcpy_chk_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, __memcpy_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memcpy_chk_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, __memcpy_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memcpy_chk_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, __memcpy_chk,
                              HAS_ARCH_FEATURE (AVX_Usable),
@@ -802,10 +802,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __memcpy_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, memcpy,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memcpy_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, memcpy,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __memcpy_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
              IFUNC_IMPL_ADD (array, i, memcpy, 1,
@@ -819,10 +819,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __mempcpy_chk_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __mempcpy_chk_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __mempcpy_chk_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
                              HAS_ARCH_FEATURE (AVX_Usable),
@@ -864,10 +864,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
                              HAS_ARCH_FEATURE (AVX512F_Usable),
                              __mempcpy_avx512_no_vzeroupper)
              IFUNC_IMPL_ADD (array, i, mempcpy,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __mempcpy_avx512_unaligned)
              IFUNC_IMPL_ADD (array, i, mempcpy,
-                             HAS_ARCH_FEATURE (AVX512F_Usable),
+                             HAS_ARCH_FEATURE (AVX512VL_Usable),
                              __mempcpy_avx512_unaligned_erms)
              IFUNC_IMPL_ADD (array, i, mempcpy,
                              HAS_ARCH_FEATURE (AVX_Usable),

diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
index a91c1f27cadae182f653f7e36af0fa4d4114c1fd..01fc6b998b4d113b70ea2c9a0a10063894f35cea 100644 (file)
--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
@@ -56,13 +56,15 @@ IFUNC_SELECTOR (void)
   if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
       && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
     {
-      if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
-       return OPTIMIZE (avx512_no_vzeroupper);
+      if (CPU_FEATURES_ARCH_P (cpu_features, AVX512VL_Usable))
+       {
+       if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
+           return OPTIMIZE (avx512_unaligned_erms);
 
-      if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
-       return OPTIMIZE (avx512_unaligned_erms);
+         return OPTIMIZE (avx512_unaligned);
+       }
 
-      return OPTIMIZE (avx512_unaligned);
+      return OPTIMIZE (avx512_no_vzeroupper);
     }
 
   if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))

diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
index aac1515cf6b3bb5c29377de6ffdafa89641cd7ad..7dad1ad74c9afe1952d1dc98a3970f15df990c66 100644 (file)
--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
@@ -1,11 +1,25 @@
 #if IS_IN (libc)
 # define VEC_SIZE      64
-# define VEC(i)                zmm##i
+# define XMM0          xmm16
+# define XMM1          xmm17
+# define YMM0          ymm16
+# define YMM1          ymm17
+# define VEC0          zmm16
+# define VEC1          zmm17
+# define VEC2          zmm18
+# define VEC3          zmm19
+# define VEC4          zmm20
+# define VEC5          zmm21
+# define VEC6          zmm22
+# define VEC7          zmm23
+# define VEC8          zmm24
+# define VEC(i)                VEC##i
 # define VMOVNT                vmovntdq
 # define VMOVU         vmovdqu64
 # define VMOVA         vmovdqa64
+# define VZEROUPPER
 
-# define SECTION(p)            p##.avx512
+# define SECTION(p)            p##.evex512
 # define MEMMOVE_SYMBOL(p,s)   p##_avx512_##s
 
 # include "memmove-vec-unaligned-erms.S"