Implement x86-64 multiarch mempcpy in memcpy

Implement x86-64 multiarch mempcpy in memcpy to share most of code.  It
reduces code size of libc.so.

	[BZ #18858]
	* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Remove
	mempcpy-ssse3, mempcpy-ssse3-back, mempcpy-avx-unaligned
	and mempcpy-avx512-no-vzeroupper.
	* sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S (MEMPCPY_CHK):
	New.
	(MEMPCPY): Likewise.
	* sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S
	(MEMPCPY_CHK): New.
	(MEMPCPY): Likewise.
	* sysdeps/x86_64/multiarch/memcpy-ssse3-back.S (MEMPCPY_CHK): New.
	(MEMPCPY): Likewise.
	* sysdeps/x86_64/multiarch/memcpy-ssse3.S (MEMPCPY_CHK): New.
	(MEMPCPY): Likewise.
	* sysdeps/x86_64/multiarch/mempcpy-avx-unaligned.S: Removed.
	* sysdeps/x86_64/multiarch/mempcpy-avx512-no-vzeroupper.S:
	Likewise.
	* sysdeps/x86_64/multiarch/mempcpy-ssse3-back.S: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy-ssse3.S: Likewise.

diff --git a/ChangeLog b/ChangeLog
index 5375f3b508f20ae9e0e88fd0ae390dd6a74b5878..b7a07a06e8ea9cb378198db39afc47f47de2618a 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,25 @@
+2016-03-28   H.J. Lu  <hongjiu.lu@intel.com>
+
+       [BZ #18858]
+       * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Remove
+       mempcpy-ssse3, mempcpy-ssse3-back, mempcpy-avx-unaligned
+       and mempcpy-avx512-no-vzeroupper.
+       * sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S (MEMPCPY_CHK):
+       New.
+       (MEMPCPY): Likewise.
+       * sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S
+       (MEMPCPY_CHK): New.
+       (MEMPCPY): Likewise.
+       * sysdeps/x86_64/multiarch/memcpy-ssse3-back.S (MEMPCPY_CHK): New.
+       (MEMPCPY): Likewise.
+       * sysdeps/x86_64/multiarch/memcpy-ssse3.S (MEMPCPY_CHK): New.
+       (MEMPCPY): Likewise.
+       * sysdeps/x86_64/multiarch/mempcpy-avx-unaligned.S: Removed.
+       * sysdeps/x86_64/multiarch/mempcpy-avx512-no-vzeroupper.S:
+       Likewise.
+       * sysdeps/x86_64/multiarch/mempcpy-ssse3-back.S: Likewise.
+       * sysdeps/x86_64/multiarch/mempcpy-ssse3.S: Likewise.
+
 2016-03-28   H.J. Lu  <hongjiu.lu@intel.com>
             Amit Pawar  <Amit.Pawar@amd.com>
 

diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index d234f4ab66e6f613aed48b2be58e4853f45e1184..39c090570e68b414aa72c0e6efc7d50b61008ec0 100644 (file)
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -8,10 +8,10 @@ ifeq ($(subdir),string)
 sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
                   strcmp-sse2-unaligned strncmp-ssse3 \
                   memcmp-sse4 memcpy-ssse3 memcpy-sse2-unaligned \
-                  memcpy-avx512-no-vzeroupper mempcpy-ssse3 memmove-ssse3 \
-                  memcpy-ssse3-back mempcpy-ssse3-back memmove-avx-unaligned \
-                  memcpy-avx-unaligned mempcpy-avx-unaligned \
-                  mempcpy-avx512-no-vzeroupper memmove-ssse3-back \
+                  memcpy-avx512-no-vzeroupper memmove-ssse3 \
+                  memcpy-ssse3-back memmove-avx-unaligned \
+                  memcpy-avx-unaligned \
+                  memmove-ssse3-back \
                   memmove-avx512-no-vzeroupper strcasecmp_l-ssse3 \
                   strncase_l-ssse3 strcat-ssse3 strncat-ssse3\
                   strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \

diff --git a/sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S b/sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S
index b615d063c073ba51b90095baf449db3da31c5366..dd4187fa36e20b77a000131c08d35586e423046d 100644 (file)
--- a/sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S
+++ b/sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S
@@ -25,11 +25,26 @@
 
 #include "asm-syntax.h"
 #ifndef MEMCPY
-# define MEMCPY        __memcpy_avx_unaligned
+# define MEMCPY                __memcpy_avx_unaligned
 # define MEMCPY_CHK    __memcpy_chk_avx_unaligned
+# define MEMPCPY       __mempcpy_avx_unaligned
+# define MEMPCPY_CHK   __mempcpy_chk_avx_unaligned
 #endif
 
        .section .text.avx,"ax",@progbits
+#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
+ENTRY (MEMPCPY_CHK)
+       cmpq    %rdx, %rcx
+       jb      HIDDEN_JUMPTARGET (__chk_fail)
+END (MEMPCPY_CHK)
+
+ENTRY (MEMPCPY)
+       movq    %rdi, %rax
+       addq    %rdx, %rax
+       jmp     L(start)
+END (MEMPCPY)
+#endif
+
 #if !defined USE_AS_BCOPY
 ENTRY (MEMCPY_CHK)
        cmpq    %rdx, %rcx
@@ -42,6 +57,7 @@ ENTRY (MEMCPY)
 #ifdef USE_AS_MEMPCPY
        add     %rdx, %rax
 #endif
+L(start):
        cmp     $256, %rdx
        jae     L(256bytesormore)
        cmp     $16, %dl

diff --git a/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S
index 3d567fc8e56394d1d7de084aacb2e1b55ed7c810..285bb838335aeba61ecda90766c566b338be0d7e 100644 (file)
--- a/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S
+++ b/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S
@@ -27,9 +27,24 @@
 #ifndef MEMCPY
 # define MEMCPY                __memcpy_avx512_no_vzeroupper
 # define MEMCPY_CHK    __memcpy_chk_avx512_no_vzeroupper
+# define MEMPCPY       __mempcpy_avx512_no_vzeroupper
+# define MEMPCPY_CHK   __mempcpy_chk_avx512_no_vzeroupper
 #endif
 
        .section .text.avx512,"ax",@progbits
+#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
+ENTRY (MEMPCPY_CHK)
+       cmpq    %rdx, %rcx
+       jb      HIDDEN_JUMPTARGET (__chk_fail)
+END (MEMPCPY_CHK)
+
+ENTRY (MEMPCPY)
+       movq    %rdi, %rax
+       addq    %rdx, %rax
+       jmp     L(start)
+END (MEMPCPY)
+#endif
+
 #if !defined USE_AS_BCOPY
 ENTRY (MEMCPY_CHK)
        cmpq    %rdx, %rcx
@@ -42,6 +57,7 @@ ENTRY (MEMCPY)
 #ifdef USE_AS_MEMPCPY
        add     %rdx, %rax
 #endif
+L(start):
        lea     (%rsi, %rdx), %rcx
        lea     (%rdi, %rdx), %r9
        cmp     $512, %rdx

diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
index 08b41e9e5a94a3afc4355a8c9394f8c64a6edc85..b4890f4da97bbf6aee9eea23e23f172ed23d5cde 100644 (file)
--- a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
+++ b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
@@ -29,6 +29,8 @@
 #ifndef MEMCPY
 # define MEMCPY                __memcpy_ssse3_back
 # define MEMCPY_CHK    __memcpy_chk_ssse3_back
+# define MEMPCPY       __mempcpy_ssse3_back
+# define MEMPCPY_CHK   __mempcpy_chk_ssse3_back
 #endif
 
 #define JMPTBL(I, B)   I - B
@@ -44,6 +46,19 @@
   ud2
 
        .section .text.ssse3,"ax",@progbits
+#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
+ENTRY (MEMPCPY_CHK)
+       cmpq    %rdx, %rcx
+       jb      HIDDEN_JUMPTARGET (__chk_fail)
+END (MEMPCPY_CHK)
+
+ENTRY (MEMPCPY)
+       movq    %rdi, %rax
+       addq    %rdx, %rax
+       jmp     L(start)
+END (MEMPCPY)
+#endif
+
 #if !defined USE_AS_BCOPY
 ENTRY (MEMCPY_CHK)
        cmpq    %rdx, %rcx
@@ -66,6 +81,7 @@ ENTRY (MEMCPY)
        BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4)
 L(copy_forward):
 #endif
+L(start):
        cmp     $144, %rdx
        jae     L(144bytesormore)
 

diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S
index 95de9695f946ea2669c87abdc56dbf6ab7ddc611..1ca88c0758863f818376098f8648e4a0b2e042c5 100644 (file)
--- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S
+++ b/sysdeps/x86_64/multiarch/memcpy-ssse3.S
@@ -29,6 +29,8 @@
 #ifndef MEMCPY
 # define MEMCPY                __memcpy_ssse3
 # define MEMCPY_CHK    __memcpy_chk_ssse3
+# define MEMPCPY       __mempcpy_ssse3
+# define MEMPCPY_CHK   __mempcpy_chk_ssse3
 #endif
 
 #define JMPTBL(I, B)   I - B
@@ -44,6 +46,19 @@
   ud2
 
        .section .text.ssse3,"ax",@progbits
+#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
+ENTRY (MEMPCPY_CHK)
+       cmpq    %rdx, %rcx
+       jb      HIDDEN_JUMPTARGET (__chk_fail)
+END (MEMPCPY_CHK)
+
+ENTRY (MEMPCPY)
+       movq    %rdi, %rax
+       addq    %rdx, %rax
+       jmp     L(start)
+END (MEMPCPY)
+#endif
+
 #if !defined USE_AS_BCOPY
 ENTRY (MEMCPY_CHK)
        cmpq    %rdx, %rcx
@@ -66,6 +81,7 @@ ENTRY (MEMCPY)
        jmp     L(copy_backward)
 L(copy_forward):
 #endif
+L(start):
        cmp     $79, %rdx
        lea     L(table_less_80bytes)(%rip), %r11
        ja      L(80bytesormore)

diff --git a/sysdeps/x86_64/multiarch/mempcpy-avx-unaligned.S b/sysdeps/x86_64/multiarch/mempcpy-avx-unaligned.S
deleted file mode 100644 (file)
index 241378e..0000000
--- a/sysdeps/x86_64/multiarch/mempcpy-avx-unaligned.S
+++ /dev/null
@@ -1,22 +0,0 @@
-/* mempcpy with AVX
-   Copyright (C) 2014-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#define USE_AS_MEMPCPY
-#define MEMCPY         __mempcpy_avx_unaligned
-#define MEMCPY_CHK     __mempcpy_chk_avx_unaligned
-#include "memcpy-avx-unaligned.S"

diff --git a/sysdeps/x86_64/multiarch/mempcpy-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/mempcpy-avx512-no-vzeroupper.S
deleted file mode 100644 (file)
index fcc0945..0000000
--- a/sysdeps/x86_64/multiarch/mempcpy-avx512-no-vzeroupper.S
+++ /dev/null
@@ -1,22 +0,0 @@
-/* mempcpy optimized with AVX512 for KNL hardware.
-   Copyright (C) 2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#define USE_AS_MEMPCPY
-#define MEMCPY         __mempcpy_avx512_no_vzeroupper
-#define MEMCPY_CHK     __mempcpy_chk_avx512_no_vzeroupper
-#include "memcpy-avx512-no-vzeroupper.S"

diff --git a/sysdeps/x86_64/multiarch/mempcpy-ssse3-back.S b/sysdeps/x86_64/multiarch/mempcpy-ssse3-back.S
deleted file mode 100644 (file)
index 82ffacb..0000000
--- a/sysdeps/x86_64/multiarch/mempcpy-ssse3-back.S
+++ /dev/null
@@ -1,4 +0,0 @@
-#define USE_AS_MEMPCPY
-#define MEMCPY         __mempcpy_ssse3_back
-#define MEMCPY_CHK     __mempcpy_chk_ssse3_back
-#include "memcpy-ssse3-back.S"

diff --git a/sysdeps/x86_64/multiarch/mempcpy-ssse3.S b/sysdeps/x86_64/multiarch/mempcpy-ssse3.S
deleted file mode 100644 (file)
index 822d98e..0000000
--- a/sysdeps/x86_64/multiarch/mempcpy-ssse3.S
+++ /dev/null
@@ -1,4 +0,0 @@
-#define USE_AS_MEMPCPY
-#define MEMCPY         __mempcpy_ssse3
-#define MEMCPY_CHK     __mempcpy_chk_ssse3
-#include "memcpy-ssse3.S"