/* Machine-dependent ELF dynamic relocation inline functions. x86-64 version.
- Copyright (C) 2001-2012 Free Software Foundation, Inc.
+ Copyright (C) 2001-2019 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Andreas Jaeger <aj@suse.de>.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
+ <https://www.gnu.org/licenses/>. */
#ifndef dl_machine_h
#define dl_machine_h
#include <sysdep.h>
#include <tls.h>
#include <dl-tlsdesc.h>
+#include <cpu-features.c>
/* Return nonzero iff ELF header is compatible with the running host. */
static inline int __attribute__ ((unused))
static inline ElfW(Addr) __attribute__ ((unused))
elf_machine_dynamic (void)
{
- ElfW(Addr) addr;
-
- /* This works because we have our GOT address available in the small PIC
- model. */
- addr = (ElfW(Addr)) &_DYNAMIC;
-
- return addr;
+ /* This produces an IP-relative reloc which is resolved at link time. */
+ extern const ElfW(Addr) _GLOBAL_OFFSET_TABLE_[] attribute_hidden;
+ return _GLOBAL_OFFSET_TABLE_[0];
}
static inline ElfW(Addr) __attribute__ ((unused))
elf_machine_load_address (void)
{
- ElfW(Addr) addr;
-
- /* The easy way is just the same as on x86:
- leaq _dl_start, %0
- leaq _dl_start(%%rip), %1
- subq %0, %1
- but this does not work with binutils since we then have
- a R_X86_64_32S relocation in a shared lib.
-
- Instead we store the address of _dl_start in the data section
- and compare it with the current value that we can get via
- an RIP relative addressing mode. Note that this is the address
- of _dl_start before any relocation performed at runtime. In case
- the binary is prelinked the resulting "address" is actually a
- load offset which is zero if the binary was loaded at the address
- it is prelinked for. */
-
- asm ("leaq _dl_start(%%rip), %0\n\t"
- "subq 1f(%%rip), %0\n\t"
- ".section\t.data.rel.ro\n"
- "1:\t.quad _dl_start\n\t"
- ".previous\n\t"
- : "=r" (addr) : : "cc");
-
- return addr;
+ /* Compute the difference between the runtime address of _DYNAMIC as seen
+ by an IP-relative reference, and the link-time address found in the
+ special unrelocated first GOT entry. */
+ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
+ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic ();
}
/* Set up the loaded object described by L so its unrelocated PLT
elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
{
Elf64_Addr *got;
- extern void _dl_runtime_resolve (ElfW(Word)) attribute_hidden;
- extern void _dl_runtime_profile (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_fxsave (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_xsave (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_xsavec (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_profile_sse (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_profile_avx (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_profile_avx512 (ElfW(Word)) attribute_hidden;
if (l->l_info[DT_JMPREL] && lazy)
{
to intercept the calls to collect information. In this case we
don't store the address in the GOT so that all future calls also
end in this function. */
- if (__builtin_expect (profile, 0))
+ if (__glibc_unlikely (profile))
{
- *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile;
+ if (HAS_ARCH_FEATURE (AVX512F_Usable))
+ *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx512;
+ else if (HAS_ARCH_FEATURE (AVX_Usable))
+ *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx;
+ else
+ *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_sse;
if (GLRO(dl_profile) != NULL
&& _dl_name_match_p (GLRO(dl_profile), l))
GL(dl_profile_map) = l;
}
else
- /* This function will get called to fix up the GOT entry indicated by
- the offset on the stack, and then jump to the resolved address. */
- *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_resolve;
+ {
+ /* This function will get called to fix up the GOT entry
+ indicated by the offset on the stack, and then jump to
+ the resolved address. */
+ if (GLRO(dl_x86_cpu_features).xsave_state_size != 0)
+ *(ElfW(Addr) *) (got + 2)
+ = (HAS_ARCH_FEATURE (XSAVEC_Usable)
+ ? (ElfW(Addr)) &_dl_runtime_resolve_xsavec
+ : (ElfW(Addr)) &_dl_runtime_resolve_xsave);
+ else
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_fxsave;
+ }
}
if (l->l_info[ADDRIDX (DT_TLSDESC_GOT)] && lazy)
movq %rdx, %rsi\n\
# Save %rsp value in %r13.\n\
movq %rsp, %r13\n\
- # And align stack for the _dl_init_internal call. \n\
+ # And align stack for the _dl_init call. \n\
andq $-16, %rsp\n\
# _dl_loaded -> rdi\n\
movq _rtld_local(%rip), %rdi\n\
# Clear %rbp to mark outermost frame obviously even for constructors.\n\
xorl %ebp, %ebp\n\
# Call the function to run the initializers.\n\
- call _dl_init_internal@PLT\n\
+ call _dl_init\n\
# Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\
leaq _dl_fini(%rip), %rdx\n\
# And make sure %rsp points to argc stored on the stack.\n\
/* ELF_RTYPE_CLASS_PLT iff TYPE describes relocation of a PLT entry or
TLS variable, so undefined references should not be allowed to
define the value.
- ELF_RTYPE_CLASS_NOCOPY iff TYPE should not be allowed to resolve to one
- of the main executable's symbols, as for a COPY reloc. */
+ ELF_RTYPE_CLASS_COPY iff TYPE should not be allowed to resolve to one
+ of the main executable's symbols, as for a COPY reloc.
+ ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA iff TYPE describes relocation may
+ against protected data whose address be external due to copy relocation.
+ */
#define elf_machine_type_class(type) \
((((type) == R_X86_64_JUMP_SLOT \
|| (type) == R_X86_64_DTPMOD64 \
|| (type) == R_X86_64_TPOFF64 \
|| (type) == R_X86_64_TLSDESC) \
* ELF_RTYPE_CLASS_PLT) \
- | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY))
+ | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY) \
+ | (((type) == R_X86_64_GLOB_DAT) * ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA))
/* A reloc type used for ld.so cmdline arg lookups to reject PLT entries. */
#define ELF_MACHINE_JMP_SLOT R_X86_64_JUMP_SLOT
/* The x86-64 never uses Elf64_Rel/Elf32_Rel relocations. */
#define ELF_MACHINE_NO_REL 1
+#define ELF_MACHINE_NO_RELA 0
/* We define an initialization function. This is called very early in
_dl_sysdep_start. */
static inline void __attribute__ ((unused))
dl_platform_init (void)
{
+#if IS_IN (rtld)
+ /* init_cpu_features has been called early from __libc_start_main in
+ static executable. */
+ init_cpu_features (&GLRO(dl_x86_cpu_features));
+#else
if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
/* Avoid an empty string which would disturb us. */
GLRO(dl_platform) = NULL;
+#endif
}
static inline ElfW(Addr)
elf_machine_fixup_plt (struct link_map *map, lookup_t t,
+ const ElfW(Sym) *refsym, const ElfW(Sym) *sym,
const ElfW(Rela) *reloc,
ElfW(Addr) *reloc_addr, ElfW(Addr) value)
{
const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
# if !defined RTLD_BOOTSTRAP || !defined HAVE_Z_COMBRELOC
- if (__builtin_expect (r_type == R_X86_64_RELATIVE, 0))
+ if (__glibc_unlikely (r_type == R_X86_64_RELATIVE))
{
# if !defined RTLD_BOOTSTRAP && !defined HAVE_Z_COMBRELOC
/* This is defined in rtld.c, but nowhere in the static libc.a;
}
else
# endif
- if (__builtin_expect (r_type == R_X86_64_NONE, 0))
+# if !defined RTLD_BOOTSTRAP
+ /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
+ relocation updates the whole 64-bit entry. */
+ if (__glibc_unlikely (r_type == R_X86_64_RELATIVE64))
+ *(Elf64_Addr *) reloc_addr = (Elf64_Addr) map->l_addr + reloc->r_addend;
+ else
+# endif
+ if (__glibc_unlikely (r_type == R_X86_64_NONE))
return;
else
{
const ElfW(Sym) *const refsym = sym;
# endif
struct link_map *sym_map = RESOLVE_MAP (&sym, version, r_type);
- ElfW(Addr) value = (sym == NULL ? 0
- : (ElfW(Addr)) sym_map->l_addr + sym->st_value);
+ ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);
if (sym != NULL
- && __builtin_expect (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC,
- 0)
- && __builtin_expect (sym->st_shndx != SHN_UNDEF, 1)
- && __builtin_expect (!skip_ifunc, 1))
- value = ((ElfW(Addr) (*) (void)) value) ();
+ && __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
+ && __glibc_likely (sym->st_shndx != SHN_UNDEF)
+ && __glibc_likely (!skip_ifunc))
+ {
+# ifndef RTLD_BOOTSTRAP
+ if (sym_map != map
+ && sym_map->l_type != lt_executable
+ && !sym_map->l_relocated)
+ {
+ const char *strtab
+ = (const char *) D_PTR (map, l_info[DT_STRTAB]);
+ _dl_error_printf ("\
+%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
+ RTLD_PROGNAME, map->l_name,
+ sym_map->l_name,
+ strtab + refsym->st_name);
+ }
+# endif
+ value = ((ElfW(Addr) (*) (void)) value) ();
+ }
switch (r_type)
{
+# ifndef RTLD_BOOTSTRAP
+# ifdef __ILP32__
+ case R_X86_64_SIZE64:
+ /* Set to symbol size plus addend. */
+ *(Elf64_Addr *) (uintptr_t) reloc_addr
+ = (Elf64_Addr) sym->st_size + reloc->r_addend;
+ break;
+
+ case R_X86_64_SIZE32:
+# else
+ case R_X86_64_SIZE64:
+# endif
+ /* Set to symbol size plus addend. */
+ value = sym->st_size;
+# endif
+ /* Fall through. */
case R_X86_64_GLOB_DAT:
case R_X86_64_JUMP_SLOT:
*reloc_addr = value + reloc->r_addend;
# ifndef RTLD_BOOTSTRAP
case R_X86_64_64:
- *reloc_addr = value + reloc->r_addend;
+ /* value + r_addend may be > 0xffffffff and R_X86_64_64
+ relocation updates the whole 64-bit entry. */
+ *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend;
break;
+# ifndef __ILP32__
+ case R_X86_64_SIZE32:
+ /* Set to symbol size plus addend. */
+ value = sym->st_size;
+# endif
+ /* Fall through. */
case R_X86_64_32:
value += reloc->r_addend;
*(unsigned int *) reloc_addr = value;
const char *fmt;
- if (__builtin_expect (value > UINT_MAX, 0))
+ if (__glibc_unlikely (value > UINT_MAX))
{
const char *strtab;
# endif
strtab = (const char *) D_PTR (map, l_info[DT_STRTAB]);
- _dl_error_printf (fmt,
- rtld_progname ?: "<program name unknown>",
- strtab + refsym->st_name);
+ _dl_error_printf (fmt, RTLD_PROGNAME, strtab + refsym->st_name);
}
break;
# ifndef RESOLVE_CONFLICT_FIND_MAP
case R_X86_64_PC32:
value += reloc->r_addend - (ElfW(Addr)) reloc_addr;
*(unsigned int *) reloc_addr = value;
- if (__builtin_expect (value != (int) value, 0))
+ if (__glibc_unlikely (value != (int) value))
{
fmt = "\
%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
break;
memcpy (reloc_addr_arg, (void *) value,
MIN (sym->st_size, refsym->st_size));
- if (__builtin_expect (sym->st_size > refsym->st_size, 0)
- || (__builtin_expect (sym->st_size < refsym->st_size, 0)
+ if (__glibc_unlikely (sym->st_size > refsym->st_size)
+ || (__glibc_unlikely (sym->st_size < refsym->st_size)
&& GLRO(dl_verbose)))
{
fmt = "\
void *const reloc_addr_arg)
{
ElfW(Addr) *const reloc_addr = reloc_addr_arg;
- assert (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE);
- *reloc_addr = l_addr + reloc->r_addend;
+#if !defined RTLD_BOOTSTRAP
+ /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
+ relocation updates the whole 64-bit entry. */
+ if (__glibc_unlikely (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE64))
+ *(Elf64_Addr *) reloc_addr = (Elf64_Addr) l_addr + reloc->r_addend;
+ else
+#endif
+ {
+ assert (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE);
+ *reloc_addr = l_addr + reloc->r_addend;
+ }
}
auto inline void
const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
/* Check for unexpected PLT reloc type. */
- if (__builtin_expect (r_type == R_X86_64_JUMP_SLOT, 1))
+ if (__glibc_likely (r_type == R_X86_64_JUMP_SLOT))
{
- if (__builtin_expect (map->l_mach.plt, 0) == 0)
+ /* Prelink has been deprecated. */
+ if (__glibc_likely (map->l_mach.plt == 0))
*reloc_addr += l_addr;
else
*reloc_addr =
map->l_mach.plt
+ (((ElfW(Addr)) reloc_addr) - map->l_mach.gotplt) * 2;
}
- else if (__builtin_expect (r_type == R_X86_64_TLSDESC, 1))
+ else if (__glibc_likely (r_type == R_X86_64_TLSDESC))
{
struct tlsdesc volatile * __attribute__((__unused__)) td =
(struct tlsdesc volatile *)reloc_addr;
td->entry = (void*)(D_PTR (map, l_info[ADDRIDX (DT_TLSDESC_PLT)])
+ map->l_addr);
}
- else if (__builtin_expect (r_type == R_X86_64_IRELATIVE, 0))
+ else if (__glibc_unlikely (r_type == R_X86_64_IRELATIVE))
{
ElfW(Addr) value = map->l_addr + reloc->r_addend;
- if (__builtin_expect (!skip_ifunc, 1))
+ if (__glibc_likely (!skip_ifunc))
value = ((ElfW(Addr) (*) (void)) value) ();
*reloc_addr = value;
}
projects / thirdparty / glibc.git / blobdiff