/* PLT trampolines. x86-64 version.
- Copyright (C) 2004, 2005, 2007, 2009 Free Software Foundation, Inc.
+ Copyright (C) 2004-2015 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, write to the Free
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
- 02111-1307 USA. */
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
#include <config.h>
#include <sysdep.h>
#include <link-defines.h>
+#if (RTLD_SAVESPACE_SSE % 32) != 0
+# error RTLD_SAVESPACE_SSE must be aligned to 32 bytes
+#endif
+
+/* Area on stack to save and restore registers used for parameter
+ passing when calling _dl_fixup. */
+#ifdef __ILP32__
+/* X32 saves RCX, RDX, RSI, RDI, R8 and R9 plus RAX. */
+# define REGISTER_SAVE_AREA (8 * 7)
+# define REGISTER_SAVE_RAX 0
+#else
+/* X86-64 saves RCX, RDX, RSI, RDI, R8 and R9 plus RAX as well as BND0,
+ BND1, BND2, BND3. */
+# define REGISTER_SAVE_AREA (8 * 7 + 16 * 4)
+/* Align bound register save area to 16 bytes. */
+# define REGISTER_SAVE_BND0 0
+# define REGISTER_SAVE_BND1 (REGISTER_SAVE_BND0 + 16)
+# define REGISTER_SAVE_BND2 (REGISTER_SAVE_BND1 + 16)
+# define REGISTER_SAVE_BND3 (REGISTER_SAVE_BND2 + 16)
+# define REGISTER_SAVE_RAX (REGISTER_SAVE_BND3 + 16)
+#endif
+#define REGISTER_SAVE_RCX (REGISTER_SAVE_RAX + 8)
+#define REGISTER_SAVE_RDX (REGISTER_SAVE_RCX + 8)
+#define REGISTER_SAVE_RSI (REGISTER_SAVE_RDX + 8)
+#define REGISTER_SAVE_RDI (REGISTER_SAVE_RSI + 8)
+#define REGISTER_SAVE_R8 (REGISTER_SAVE_RDI + 8)
+#define REGISTER_SAVE_R9 (REGISTER_SAVE_R8 + 8)
+
.text
.globl _dl_runtime_resolve
.type _dl_runtime_resolve, @function
.align 16
cfi_startproc
_dl_runtime_resolve:
- subq $56,%rsp
- cfi_adjust_cfa_offset(72) # Incorporate PLT
- movq %rax,(%rsp) # Preserve registers otherwise clobbered.
- movq %rcx, 8(%rsp)
- movq %rdx, 16(%rsp)
- movq %rsi, 24(%rsp)
- movq %rdi, 32(%rsp)
- movq %r8, 40(%rsp)
- movq %r9, 48(%rsp)
- movq 64(%rsp), %rsi # Copy args pushed by PLT in register.
- movq 56(%rsp), %rdi # %rdi: link_map, %rsi: reloc_index
+ cfi_adjust_cfa_offset(16) # Incorporate PLT
+ subq $REGISTER_SAVE_AREA,%rsp
+ cfi_adjust_cfa_offset(REGISTER_SAVE_AREA)
+ # Preserve registers otherwise clobbered.
+ movq %rax, REGISTER_SAVE_RAX(%rsp)
+ movq %rcx, REGISTER_SAVE_RCX(%rsp)
+ movq %rdx, REGISTER_SAVE_RDX(%rsp)
+ movq %rsi, REGISTER_SAVE_RSI(%rsp)
+ movq %rdi, REGISTER_SAVE_RDI(%rsp)
+ movq %r8, REGISTER_SAVE_R8(%rsp)
+ movq %r9, REGISTER_SAVE_R9(%rsp)
+#ifndef __ILP32__
+ # We also have to preserve bound registers. These are nops if
+ # Intel MPX isn't available or disabled.
+# ifdef HAVE_MPX_SUPPORT
+ bndmov %bnd0, REGISTER_SAVE_BND0(%rsp)
+ bndmov %bnd1, REGISTER_SAVE_BND1(%rsp)
+ bndmov %bnd2, REGISTER_SAVE_BND2(%rsp)
+ bndmov %bnd3, REGISTER_SAVE_BND3(%rsp)
+# else
+ .byte 0x66,0x0f,0x1b,0x44,0x24,REGISTER_SAVE_BND0
+ .byte 0x66,0x0f,0x1b,0x4c,0x24,REGISTER_SAVE_BND1
+ .byte 0x66,0x0f,0x1b,0x54,0x24,REGISTER_SAVE_BND2
+ .byte 0x66,0x0f,0x1b,0x5c,0x24,REGISTER_SAVE_BND3
+# endif
+#endif
+ # Copy args pushed by PLT in register.
+ # %rdi: link_map, %rsi: reloc_index
+ movq (REGISTER_SAVE_AREA + 8)(%rsp), %rsi
+ movq REGISTER_SAVE_AREA(%rsp), %rdi
call _dl_fixup # Call resolver.
movq %rax, %r11 # Save return value
- movq 48(%rsp), %r9 # Get register content back.
- movq 40(%rsp), %r8
- movq 32(%rsp), %rdi
- movq 24(%rsp), %rsi
- movq 16(%rsp), %rdx
- movq 8(%rsp), %rcx
- movq (%rsp), %rax
- addq $72, %rsp # Adjust stack(PLT did 2 pushes)
- cfi_adjust_cfa_offset(-72)
+#ifndef __ILP32__
+ # Restore bound registers. These are nops if Intel MPX isn't
+ # avaiable or disabled.
+# ifdef HAVE_MPX_SUPPORT
+ bndmov REGISTER_SAVE_BND3(%rsp), %bnd3
+ bndmov REGISTER_SAVE_BND2(%rsp), %bnd2
+ bndmov REGISTER_SAVE_BND1(%rsp), %bnd1
+ bndmov REGISTER_SAVE_BND0(%rsp), %bnd0
+# else
+ .byte 0x66,0x0f,0x1a,0x5c,0x24,REGISTER_SAVE_BND3
+ .byte 0x66,0x0f,0x1a,0x54,0x24,REGISTER_SAVE_BND2
+ .byte 0x66,0x0f,0x1a,0x4c,0x24,REGISTER_SAVE_BND1
+ .byte 0x66,0x0f,0x1a,0x44,0x24,REGISTER_SAVE_BND0
+# endif
+#endif
+ # Get register content back.
+ movq REGISTER_SAVE_R9(%rsp), %r9
+ movq REGISTER_SAVE_R8(%rsp), %r8
+ movq REGISTER_SAVE_RDI(%rsp), %rdi
+ movq REGISTER_SAVE_RSI(%rsp), %rsi
+ movq REGISTER_SAVE_RDX(%rsp), %rdx
+ movq REGISTER_SAVE_RCX(%rsp), %rcx
+ movq REGISTER_SAVE_RAX(%rsp), %rax
+ # Adjust stack(PLT did 2 pushes)
+ addq $(REGISTER_SAVE_AREA + 16), %rsp
+ cfi_adjust_cfa_offset(-(REGISTER_SAVE_AREA + 16))
jmp *%r11 # Jump to function address.
cfi_endproc
.size _dl_runtime_resolve, .-_dl_runtime_resolve
cfi_startproc
_dl_runtime_profile:
+ cfi_adjust_cfa_offset(16) # Incorporate PLT
/* The La_x86_64_regs data structure pointed to by the
fourth paramater must be 16-byte aligned. This must
be explicitly enforced. We have the set up a dynamically
has a fixed size and preserves the original stack pointer. */
subq $32, %rsp # Allocate the local storage.
- cfi_adjust_cfa_offset(48) # Incorporate PLT
+ cfi_adjust_cfa_offset(32)
movq %rbx, (%rsp)
cfi_rel_offset(%rbx, 0)
/* Actively align the La_x86_64_regs structure. */
andq $0xfffffffffffffff0, %rsp
-# ifdef HAVE_AVX_SUPPORT
+# if defined HAVE_AVX_SUPPORT || defined HAVE_AVX512_ASM_SUPPORT
/* sizeof(La_x86_64_regs). Need extra space for 8 SSE registers
to detect if any xmm0-xmm7 registers are changed by audit
module. */
movq %rax, LR_RSP_OFFSET(%rsp)
/* We always store the XMM registers even if AVX is available.
- This is to provide backward binary compatility for existing
+ This is to provide backward binary compatibility for existing
audit modules. */
movaps %xmm0, (LR_XMM_OFFSET)(%rsp)
movaps %xmm1, (LR_XMM_OFFSET + XMM_SIZE)(%rsp)
movaps %xmm6, (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp)
movaps %xmm7, (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp)
-# ifdef HAVE_AVX_SUPPORT
+# ifndef __ILP32__
+# ifdef HAVE_MPX_SUPPORT
+ bndmov %bnd0, (LR_BND_OFFSET)(%rsp) # Preserve bound
+ bndmov %bnd1, (LR_BND_OFFSET + BND_SIZE)(%rsp) # registers. Nops if
+ bndmov %bnd2, (LR_BND_OFFSET + BND_SIZE*2)(%rsp) # MPX not available
+ bndmov %bnd3, (LR_BND_OFFSET + BND_SIZE*3)(%rsp) # or disabled.
+# else
+ .byte 0x66,0x0f,0x1b,0x84,0x24;.long (LR_BND_OFFSET)
+ .byte 0x66,0x0f,0x1b,0x8c,0x24;.long (LR_BND_OFFSET + BND_SIZE)
+ .byte 0x66,0x0f,0x1b,0x84,0x24;.long (LR_BND_OFFSET + BND_SIZE*2)
+ .byte 0x66,0x0f,0x1b,0x8c,0x24;.long (LR_BND_OFFSET + BND_SIZE*3)
+# endif
+# endif
+
+# if defined HAVE_AVX_SUPPORT || defined HAVE_AVX512_ASM_SUPPORT
.data
L(have_avx):
.zero 4
.previous
cmpl $0, L(have_avx)(%rip)
- jne 1f
+ jne L(defined)
movq %rbx, %r11 # Save rbx
movl $1, %eax
cpuid
movq %r11,%rbx # Restore rbx
- movl $1, %eax
- testl $(1 << 28), %ecx
- jne 2f
- negl %eax
-2: movl %eax, L(have_avx)(%rip)
+ xorl %eax, %eax
+ // AVX and XSAVE supported?
+ andl $((1 << 28) | (1 << 27)), %ecx
+ cmpl $((1 << 28) | (1 << 27)), %ecx
+ jne 10f
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ // AVX512 supported in processor?
+ movq %rbx, %r11 # Save rbx
+ xorl %ecx, %ecx
+ mov $0x7, %eax
+ cpuid
+ andl $(1 << 16), %ebx
+# endif
+ xorl %ecx, %ecx
+ // Get XFEATURE_ENABLED_MASK
+ xgetbv
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ test %ebx, %ebx
+ movq %r11, %rbx # Restore rbx
+ je 20f
+ // Verify that XCR0[7:5] = '111b' and
+ // XCR0[2:1] = '11b' which means
+ // that zmm state is enabled
+ andl $0xe6, %eax
+ cmpl $0xe6, %eax
+ jne 20f
+ movl %eax, L(have_avx)(%rip)
+L(avx512):
+# define RESTORE_AVX
+# define VMOV vmovdqu64
+# define VEC(i) zmm##i
+# define MORE_CODE
+# include "dl-trampoline.h"
+# undef VMOV
+# undef VEC
+# undef RESTORE_AVX
+# endif
+20: andl $0x6, %eax
+10: subl $0x5, %eax
+ movl %eax, L(have_avx)(%rip)
cmpl $0, %eax
-1: js L(no_avx1)
-
- /* This is to support AVX audit modules. */
- vmovdqu %ymm0, (LR_VECTOR_OFFSET)(%rsp)
- vmovdqu %ymm1, (LR_VECTOR_OFFSET + VECTOR_SIZE)(%rsp)
- vmovdqu %ymm2, (LR_VECTOR_OFFSET + VECTOR_SIZE*2)(%rsp)
- vmovdqu %ymm3, (LR_VECTOR_OFFSET + VECTOR_SIZE*3)(%rsp)
- vmovdqu %ymm4, (LR_VECTOR_OFFSET + VECTOR_SIZE*4)(%rsp)
- vmovdqu %ymm5, (LR_VECTOR_OFFSET + VECTOR_SIZE*5)(%rsp)
- vmovdqu %ymm6, (LR_VECTOR_OFFSET + VECTOR_SIZE*6)(%rsp)
- vmovdqu %ymm7, (LR_VECTOR_OFFSET + VECTOR_SIZE*7)(%rsp)
-
- /* Save xmm0-xmm7 registers to detect if any of them are
- changed by audit module. */
- vmovdqa %xmm0, (LR_SIZE)(%rsp)
- vmovdqa %xmm1, (LR_SIZE + XMM_SIZE)(%rsp)
- vmovdqa %xmm2, (LR_SIZE + XMM_SIZE*2)(%rsp)
- vmovdqa %xmm3, (LR_SIZE + XMM_SIZE*3)(%rsp)
- vmovdqa %xmm4, (LR_SIZE + XMM_SIZE*4)(%rsp)
- vmovdqa %xmm5, (LR_SIZE + XMM_SIZE*5)(%rsp)
- vmovdqa %xmm6, (LR_SIZE + XMM_SIZE*6)(%rsp)
- vmovdqa %xmm7, (LR_SIZE + XMM_SIZE*7)(%rsp)
-
-L(no_avx1):
-# endif
-
- movq %rsp, %rcx # La_x86_64_regs pointer to %rcx.
- movq 48(%rbx), %rdx # Load return address if needed.
- movq 40(%rbx), %rsi # Copy args pushed by PLT in register.
- movq 32(%rbx), %rdi # %rdi: link_map, %rsi: reloc_index
- leaq 16(%rbx), %r8
- call _dl_profile_fixup # Call resolver.
+L(defined):
+ js L(no_avx)
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ cmpl $0xe6, L(have_avx)(%rip)
+ je L(avx512)
+# endif
- movq %rax, %r11 # Save return value.
-
- movq 8(%rbx), %rax # Get back register content.
- movq LR_RDX_OFFSET(%rsp), %rdx
- movq LR_R8_OFFSET(%rsp), %r8
- movq LR_R9_OFFSET(%rsp), %r9
-
-# ifdef HAVE_AVX_SUPPORT
- cmpl $0, L(have_avx)(%rip)
- js L(no_avx2)
+# define RESTORE_AVX
+# define VMOV vmovdqu
+# define VEC(i) ymm##i
+# define MORE_CODE
+# include "dl-trampoline.h"
- /* Check if any xmm0-xmm7 registers are changed by audit
- module. */
- vmovdqa (LR_XMM_OFFSET)(%rsp), %xmm0
- vpcmpeqq (LR_SIZE)(%rsp), %xmm0, %xmm1
- vpmovmskb %xmm1, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET)(%rsp), %ymm0
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE)(%rsp), %xmm1
- vpcmpeqq (LR_SIZE + XMM_SIZE)(%rsp), %xmm1, %xmm2
- vpmovmskb %xmm2, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE)(%rsp), %ymm1
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*2)(%rsp), %xmm2
- vpcmpeqq (LR_SIZE + XMM_SIZE*2)(%rsp), %xmm2, %xmm3
- vpmovmskb %xmm3, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*2)(%rsp), %ymm2
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*3)(%rsp), %xmm3
- vpcmpeqq (LR_SIZE + XMM_SIZE*3)(%rsp), %xmm3, %xmm4
- vpmovmskb %xmm4, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*3)(%rsp), %ymm3
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*4)(%rsp), %xmm4
- vpcmpeqq (LR_SIZE + XMM_SIZE*4)(%rsp), %xmm4, %xmm5
- vpmovmskb %xmm5, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*4)(%rsp), %ymm4
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*5)(%rsp), %xmm5
- vpcmpeqq (LR_SIZE + XMM_SIZE*5)(%rsp), %xmm5, %xmm6
- vpmovmskb %xmm6, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*5)(%rsp), %ymm5
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp), %xmm6
- vpcmpeqq (LR_SIZE + XMM_SIZE*6)(%rsp), %xmm6, %xmm7
- vpmovmskb %xmm7, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*6)(%rsp), %ymm6
-
-1: vmovdqa (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp), %xmm7
- vpcmpeqq (LR_SIZE + XMM_SIZE*7)(%rsp), %xmm7, %xmm8
- vpmovmskb %xmm8, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu (LR_VECTOR_OFFSET + VECTOR_SIZE*7)(%rsp), %ymm7
- jmp 1f
-
-L(no_avx2):
+ .align 16
+L(no_avx):
# endif
- movaps (LR_XMM_OFFSET)(%rsp), %xmm0
- movaps (LR_XMM_OFFSET + XMM_SIZE)(%rsp), %xmm1
- movaps (LR_XMM_OFFSET + XMM_SIZE*2)(%rsp), %xmm2
- movaps (LR_XMM_OFFSET + XMM_SIZE*3)(%rsp), %xmm3
- movaps (LR_XMM_OFFSET + XMM_SIZE*4)(%rsp), %xmm4
- movaps (LR_XMM_OFFSET + XMM_SIZE*5)(%rsp), %xmm5
- movaps (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp), %xmm6
- movaps (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp), %xmm7
-
-1: movq 16(%rbx), %r10 # Anything in framesize?
- testq %r10, %r10
- jns 3f
-
- /* There's nothing in the frame size, so there
- will be no call to the _dl_call_pltexit. */
-
- /* Get back registers content. */
- movq LR_RCX_OFFSET(%rsp), %rcx
- movq LR_RSI_OFFSET(%rsp), %rsi
- movq LR_RDI_OFFSET(%rsp), %rdi
-
- movq %rbx, %rsp
- movq (%rsp), %rbx
- cfi_restore(rbx)
- cfi_def_cfa_register(%rsp)
-
- addq $48, %rsp # Adjust the stack to the return value
- # (eats the reloc index and link_map)
- cfi_adjust_cfa_offset(-48)
- jmp *%r11 # Jump to function address.
-3:
- cfi_adjust_cfa_offset(48)
- cfi_rel_offset(%rbx, 0)
- cfi_def_cfa_register(%rbx)
+# undef RESTORE_AVX
+# include "dl-trampoline.h"
- /* At this point we need to prepare new stack for the function
- which has to be called. We copy the original stack to a
- temporary buffer of the size specified by the 'framesize'
- returned from _dl_profile_fixup */
-
- leaq LR_RSP_OFFSET(%rbx), %rsi # stack
- addq $8, %r10
- andq $0xfffffffffffffff0, %r10
- movq %r10, %rcx
- subq %r10, %rsp
- movq %rsp, %rdi
- shrq $3, %rcx
- rep
- movsq
-
- movq 24(%rdi), %rcx # Get back register content.
- movq 32(%rdi), %rsi
- movq 40(%rdi), %rdi
-
- call *%r11
-
- mov 24(%rbx), %rsp # Drop the copied stack content
-
- /* Now we have to prepare the La_x86_64_retval structure for the
- _dl_call_pltexit. The La_x86_64_regs is being pointed by rsp now,
- so we just need to allocate the sizeof(La_x86_64_retval) space on
- the stack, since the alignment has already been taken care of. */
-# ifdef HAVE_AVX_SUPPORT
- /* sizeof(La_x86_64_retval). Need extra space for 2 SSE
- registers to detect if xmm0/xmm1 registers are changed
- by audit module. */
- subq $(LRV_SIZE + XMM_SIZE*2), %rsp
-# else
- subq $LRV_SIZE, %rsp # sizeof(La_x86_64_retval)
-# endif
- movq %rsp, %rcx # La_x86_64_retval argument to %rcx.
-
- /* Fill in the La_x86_64_retval structure. */
- movq %rax, LRV_RAX_OFFSET(%rcx)
- movq %rdx, LRV_RDX_OFFSET(%rcx)
+ cfi_endproc
+ .size _dl_runtime_profile, .-_dl_runtime_profile
+#endif
- movaps %xmm0, LRV_XMM0_OFFSET(%rcx)
- movaps %xmm1, LRV_XMM1_OFFSET(%rcx)
-# ifdef HAVE_AVX_SUPPORT
+#ifdef SHARED
+ .globl _dl_x86_64_save_sse
+ .type _dl_x86_64_save_sse, @function
+ .align 16
+ cfi_startproc
+_dl_x86_64_save_sse:
+# if defined HAVE_AVX_SUPPORT || defined HAVE_AVX512_ASM_SUPPORT
cmpl $0, L(have_avx)(%rip)
- js L(no_avx3)
-
- /* This is to support AVX audit modules. */
- vmovdqu %ymm0, LRV_VECTOR0_OFFSET(%rcx)
- vmovdqu %ymm1, LRV_VECTOR1_OFFSET(%rcx)
-
- /* Save xmm0/xmm1 registers to detect if they are changed
- by audit module. */
- vmovdqa %xmm0, (LRV_SIZE)(%rcx)
- vmovdqa %xmm1, (LRV_SIZE + XMM_SIZE)(%rcx)
+ jne L(defined_5)
+ movq %rbx, %r11 # Save rbx
+ movl $1, %eax
+ cpuid
+ movq %r11,%rbx # Restore rbx
+ xorl %eax, %eax
+ // AVX and XSAVE supported?
+ andl $((1 << 28) | (1 << 27)), %ecx
+ cmpl $((1 << 28) | (1 << 27)), %ecx
+ jne 1f
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ // AVX512 supported in a processor?
+ movq %rbx, %r11 # Save rbx
+ xorl %ecx,%ecx
+ mov $0x7,%eax
+ cpuid
+ andl $(1 << 16), %ebx
+# endif
+ xorl %ecx, %ecx
+ // Get XFEATURE_ENABLED_MASK
+ xgetbv
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ test %ebx, %ebx
+ movq %r11, %rbx # Restore rbx
+ je 2f
+ // Verify that XCR0[7:5] = '111b' and
+ // XCR0[2:1] = '11b' which means
+ // that zmm state is enabled
+ andl $0xe6, %eax
+ movl %eax, L(have_avx)(%rip)
+ cmpl $0xe6, %eax
+ je L(avx512_5)
+# endif
+
+2: andl $0x6, %eax
+1: subl $0x5, %eax
+ movl %eax, L(have_avx)(%rip)
+ cmpl $0, %eax
-L(no_avx3):
+L(defined_5):
+ js L(no_avx5)
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ cmpl $0xe6, L(have_avx)(%rip)
+ je L(avx512_5)
+# endif
+
+ vmovdqa %ymm0, %fs:RTLD_SAVESPACE_SSE+0*YMM_SIZE
+ vmovdqa %ymm1, %fs:RTLD_SAVESPACE_SSE+1*YMM_SIZE
+ vmovdqa %ymm2, %fs:RTLD_SAVESPACE_SSE+2*YMM_SIZE
+ vmovdqa %ymm3, %fs:RTLD_SAVESPACE_SSE+3*YMM_SIZE
+ vmovdqa %ymm4, %fs:RTLD_SAVESPACE_SSE+4*YMM_SIZE
+ vmovdqa %ymm5, %fs:RTLD_SAVESPACE_SSE+5*YMM_SIZE
+ vmovdqa %ymm6, %fs:RTLD_SAVESPACE_SSE+6*YMM_SIZE
+ vmovdqa %ymm7, %fs:RTLD_SAVESPACE_SSE+7*YMM_SIZE
+ ret
+# ifdef HAVE_AVX512_ASM_SUPPORT
+L(avx512_5):
+ vmovdqu64 %zmm0, %fs:RTLD_SAVESPACE_SSE+0*ZMM_SIZE
+ vmovdqu64 %zmm1, %fs:RTLD_SAVESPACE_SSE+1*ZMM_SIZE
+ vmovdqu64 %zmm2, %fs:RTLD_SAVESPACE_SSE+2*ZMM_SIZE
+ vmovdqu64 %zmm3, %fs:RTLD_SAVESPACE_SSE+3*ZMM_SIZE
+ vmovdqu64 %zmm4, %fs:RTLD_SAVESPACE_SSE+4*ZMM_SIZE
+ vmovdqu64 %zmm5, %fs:RTLD_SAVESPACE_SSE+5*ZMM_SIZE
+ vmovdqu64 %zmm6, %fs:RTLD_SAVESPACE_SSE+6*ZMM_SIZE
+ vmovdqu64 %zmm7, %fs:RTLD_SAVESPACE_SSE+7*ZMM_SIZE
+ ret
+# endif
+L(no_avx5):
# endif
+ movdqa %xmm0, %fs:RTLD_SAVESPACE_SSE+0*XMM_SIZE
+ movdqa %xmm1, %fs:RTLD_SAVESPACE_SSE+1*XMM_SIZE
+ movdqa %xmm2, %fs:RTLD_SAVESPACE_SSE+2*XMM_SIZE
+ movdqa %xmm3, %fs:RTLD_SAVESPACE_SSE+3*XMM_SIZE
+ movdqa %xmm4, %fs:RTLD_SAVESPACE_SSE+4*XMM_SIZE
+ movdqa %xmm5, %fs:RTLD_SAVESPACE_SSE+5*XMM_SIZE
+ movdqa %xmm6, %fs:RTLD_SAVESPACE_SSE+6*XMM_SIZE
+ movdqa %xmm7, %fs:RTLD_SAVESPACE_SSE+7*XMM_SIZE
+ ret
+ cfi_endproc
+ .size _dl_x86_64_save_sse, .-_dl_x86_64_save_sse
- fstpt LRV_ST0_OFFSET(%rcx)
- fstpt LRV_ST1_OFFSET(%rcx)
-
- movq 24(%rbx), %rdx # La_x86_64_regs argument to %rdx.
- movq 40(%rbx), %rsi # Copy args pushed by PLT in register.
- movq 32(%rbx), %rdi # %rdi: link_map, %rsi: reloc_index
- call _dl_call_pltexit
-
- /* Restore return registers. */
- movq LRV_RAX_OFFSET(%rsp), %rax
- movq LRV_RDX_OFFSET(%rsp), %rdx
-# ifdef HAVE_AVX_SUPPORT
+ .globl _dl_x86_64_restore_sse
+ .type _dl_x86_64_restore_sse, @function
+ .align 16
+ cfi_startproc
+_dl_x86_64_restore_sse:
+# if defined HAVE_AVX_SUPPORT || defined HAVE_AVX512_ASM_SUPPORT
cmpl $0, L(have_avx)(%rip)
- js L(no_avx4)
-
- /* Check if xmm0/xmm1 registers are changed by audit module. */
- vmovdqa LRV_XMM0_OFFSET(%rsp), %xmm0
- vpcmpeqq (LRV_SIZE)(%rsp), %xmm0, %xmm1
- vpmovmskb %xmm1, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu LRV_VECTOR0_OFFSET(%rsp), %ymm0
-
-1: vmovdqa LRV_XMM1_OFFSET(%rsp), %xmm1
- vpcmpeqq (LRV_SIZE + XMM_SIZE)(%rsp), %xmm1, %xmm2
- vpmovmskb %xmm2, %esi
- cmpl $0xffff, %esi
- je 1f
- vmovdqu LRV_VECTOR1_OFFSET(%rsp), %ymm1
- jmp 1f
-
-L(no_avx4):
+ js L(no_avx6)
+# ifdef HAVE_AVX512_ASM_SUPPORT
+ cmpl $0xe6, L(have_avx)(%rip)
+ je L(avx512_6)
+# endif
+
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+0*YMM_SIZE, %ymm0
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+1*YMM_SIZE, %ymm1
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+2*YMM_SIZE, %ymm2
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+3*YMM_SIZE, %ymm3
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+4*YMM_SIZE, %ymm4
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+5*YMM_SIZE, %ymm5
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+6*YMM_SIZE, %ymm6
+ vmovdqa %fs:RTLD_SAVESPACE_SSE+7*YMM_SIZE, %ymm7
+ ret
+# ifdef HAVE_AVX512_ASM_SUPPORT
+L(avx512_6):
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+0*ZMM_SIZE, %zmm0
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+1*ZMM_SIZE, %zmm1
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+2*ZMM_SIZE, %zmm2
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+3*ZMM_SIZE, %zmm3
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+4*ZMM_SIZE, %zmm4
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+5*ZMM_SIZE, %zmm5
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+6*ZMM_SIZE, %zmm6
+ vmovdqu64 %fs:RTLD_SAVESPACE_SSE+7*ZMM_SIZE, %zmm7
+ ret
+# endif
+L(no_avx6):
# endif
- movaps LRV_XMM0_OFFSET(%rsp), %xmm0
- movaps LRV_XMM1_OFFSET(%rsp), %xmm1
-
-1: fldt LRV_ST1_OFFSET(%rsp)
- fldt LRV_ST0_OFFSET(%rsp)
-
- movq %rbx, %rsp
- movq (%rsp), %rbx
- cfi_restore(rbx)
- cfi_def_cfa_register(%rsp)
-
- addq $48, %rsp # Adjust the stack to the return value
- # (eats the reloc index and link_map)
- cfi_adjust_cfa_offset(-48)
- retq
-
+ movdqa %fs:RTLD_SAVESPACE_SSE+0*XMM_SIZE, %xmm0
+ movdqa %fs:RTLD_SAVESPACE_SSE+1*XMM_SIZE, %xmm1
+ movdqa %fs:RTLD_SAVESPACE_SSE+2*XMM_SIZE, %xmm2
+ movdqa %fs:RTLD_SAVESPACE_SSE+3*XMM_SIZE, %xmm3
+ movdqa %fs:RTLD_SAVESPACE_SSE+4*XMM_SIZE, %xmm4
+ movdqa %fs:RTLD_SAVESPACE_SSE+5*XMM_SIZE, %xmm5
+ movdqa %fs:RTLD_SAVESPACE_SSE+6*XMM_SIZE, %xmm6
+ movdqa %fs:RTLD_SAVESPACE_SSE+7*XMM_SIZE, %xmm7
+ ret
cfi_endproc
- .size _dl_runtime_profile, .-_dl_runtime_profile
+ .size _dl_x86_64_restore_sse, .-_dl_x86_64_restore_sse
#endif
projects / thirdparty / glibc.git / blobdiff