[thirdparty/glibc.git] / sysdeps / aarch64 / dl-machine.h

/* Copyright (C) 1995-2019 Free Software Foundation, Inc.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public License as
   published by the Free Software Foundation; either version 2.1 of the
   License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

#ifndef dl_machine_h
#define dl_machine_h

#define ELF_MACHINE_NAME "aarch64"

#include <sysdep.h>
#include <tls.h>
#include <dl-tlsdesc.h>
#include <dl-irel.h>
#include <cpu-features.c>

/* Translate a processor specific dynamic tag to the index in l_info array.  */
#define DT_AARCH64(x) (DT_AARCH64_##x - DT_LOPROC + DT_NUM)

/* Return nonzero iff ELF header is compatible with the running host.  */
static inline int __attribute__ ((unused))
elf_machine_matches_host (const ElfW(Ehdr) *ehdr)
{
  return ehdr->e_machine == EM_AARCH64;
}

/* Return the link-time address of _DYNAMIC.  Conveniently, this is the
   first element of the GOT. */
static inline ElfW(Addr) __attribute__ ((unused))
elf_machine_dynamic (void)
{
  extern const ElfW(Addr) _GLOBAL_OFFSET_TABLE_[] attribute_hidden;
  return _GLOBAL_OFFSET_TABLE_[0];
}

/* Return the run-time load address of the shared object.  */

static inline ElfW(Addr) __attribute__ ((unused))
elf_machine_load_address (void)
{
  /* To figure out the load address we use the definition that for any symbol:
     dynamic_addr(symbol) = static_addr(symbol) + load_addr

    _DYNAMIC sysmbol is used here as its link-time address stored in
    the special unrelocated first GOT entry.  */

    extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
    return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic ();
}

/* Set up the loaded object described by L so its unrelocated PLT
   entries will jump to the on-demand fixup code in dl-runtime.c.  */

static inline int __attribute__ ((unused))
elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
{
  if (l->l_info[DT_JMPREL] && lazy)
    {
      ElfW(Addr) *got;
      extern void _dl_runtime_resolve (ElfW(Word));
      extern void _dl_runtime_profile (ElfW(Word));

      got = (ElfW(Addr) *) D_PTR (l, l_info[DT_PLTGOT]);
      if (got[1])
	{
	  l->l_mach.plt = got[1] + l->l_addr;
	}
      got[1] = (ElfW(Addr)) l;

      /* The got[2] entry contains the address of a function which gets
	 called to get the address of a so far unresolved function and
	 jump to it.  The profiling extension of the dynamic linker allows
	 to intercept the calls to collect information.  In this case we
	 don't store the address in the GOT so that all future calls also
	 end in this function.  */
      if ( profile)
	{
	   got[2] = (ElfW(Addr)) &_dl_runtime_profile;

	  if (GLRO(dl_profile) != NULL
	      && _dl_name_match_p (GLRO(dl_profile), l))
	    /* Say that we really want profiling and the timers are
	       started.  */
	    GL(dl_profile_map) = l;
	}
      else
	{
	  /* This function will get called to fix up the GOT entry
	     indicated by the offset on the stack, and then jump to
	     the resolved address.  */
	  got[2] = (ElfW(Addr)) &_dl_runtime_resolve;
	}
    }

  return lazy;
}

/* Initial entry point for the dynamic linker. The C function
   _dl_start is the real entry point, its return value is the user
   program's entry point */
#ifdef __LP64__
# define RTLD_START RTLD_START_1 ("x", "3", "sp")
#else
# define RTLD_START RTLD_START_1 ("w", "2", "wsp")
#endif


#define RTLD_START_1(PTR, PTR_SIZE_LOG, PTR_SP) asm ("\
.text									\n\
.globl _start								\n\
.type _start, %function							\n\
.globl _dl_start_user							\n\
.type _dl_start_user, %function						\n\
_start:									\n\
	mov	" PTR "0, " PTR_SP "					\n\
	bl	_dl_start						\n\
	// returns user entry point in x0				\n\
	mov	x21, x0							\n\
_dl_start_user:								\n\
	// get the original arg count					\n\
	ldr	" PTR "1, [sp]						\n\
	// get the argv address						\n\
	add	" PTR "2, " PTR_SP ", #(1<<"  PTR_SIZE_LOG ")		\n\
	// get _dl_skip_args to see if we were				\n\
	// invoked as an executable					\n\
	adrp	x4, _dl_skip_args					\n\
        ldr	w4, [x4, #:lo12:_dl_skip_args]				\n\
	// do we need to adjust argc/argv				\n\
        cmp	w4, 0							\n\
	beq	.L_done_stack_adjust					\n\
	// subtract _dl_skip_args from original arg count		\n\
	sub	" PTR "1, " PTR "1, " PTR "4				\n\
	// store adjusted argc back to stack				\n\
	str	" PTR "1, [sp]						\n\
	// find the first unskipped argument				\n\
	mov	" PTR "3, " PTR "2					\n\
	add	" PTR "4, " PTR "2, " PTR "4, lsl #" PTR_SIZE_LOG "	\n\
	// shuffle argv down						\n\
1:	ldr	" PTR "5, [x4], #(1<<"  PTR_SIZE_LOG ")			\n\
	str	" PTR "5, [x3], #(1<<"  PTR_SIZE_LOG ")			\n\
	cmp	" PTR "5, #0						\n\
	bne	1b							\n\
	// shuffle envp down						\n\
1:	ldr	" PTR "5, [x4], #(1<<"  PTR_SIZE_LOG ")			\n\
	str	" PTR "5, [x3], #(1<<"  PTR_SIZE_LOG ")			\n\
	cmp	" PTR "5, #0						\n\
	bne	1b							\n\
	// shuffle auxv down						\n\
1:	ldp	" PTR "0, " PTR "5, [x4, #(2<<"  PTR_SIZE_LOG ")]!	\n\
	stp	" PTR "0, " PTR "5, [x3], #(2<<"  PTR_SIZE_LOG ")	\n\
	cmp	" PTR "0, #0						\n\
	bne	1b							\n\
	// Update _dl_argv						\n\
	adrp	x3, __GI__dl_argv					\n\
	str	" PTR "2, [x3, #:lo12:__GI__dl_argv]			\n\
.L_done_stack_adjust:							\n\
	// compute envp							\n\
	add	" PTR "3, " PTR "2, " PTR "1, lsl #" PTR_SIZE_LOG "	\n\
	add	" PTR "3, " PTR "3, #(1<<"  PTR_SIZE_LOG ")		\n\
	adrp	x16, _rtld_local					\n\
        add	" PTR "16, " PTR "16, #:lo12:_rtld_local		\n\
        ldr	" PTR "0, [x16]						\n\
	bl	_dl_init						\n\
	// load the finalizer function					\n\
	adrp	x0, _dl_fini						\n\
	add	" PTR "0, " PTR "0, #:lo12:_dl_fini			\n\
	// jump to the user_s entry point				\n\
	br      x21							\n\
");

#define elf_machine_type_class(type)					\
  ((((type) == AARCH64_R(JUMP_SLOT)					\
     || (type) == AARCH64_R(TLS_DTPMOD)					\
     || (type) == AARCH64_R(TLS_DTPREL)					\
     || (type) == AARCH64_R(TLS_TPREL)					\
     || (type) == AARCH64_R(TLSDESC)) * ELF_RTYPE_CLASS_PLT)		\
   | (((type) == AARCH64_R(COPY)) * ELF_RTYPE_CLASS_COPY)		\
   | (((type) == AARCH64_R(GLOB_DAT)) * ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA))

#define ELF_MACHINE_JMP_SLOT	AARCH64_R(JUMP_SLOT)

/* AArch64 uses RELA not REL */
#define ELF_MACHINE_NO_REL 1
#define ELF_MACHINE_NO_RELA 0

#define DL_PLATFORM_INIT dl_platform_init ()

static inline void __attribute__ ((unused))
dl_platform_init (void)
{
  if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
    /* Avoid an empty string which would disturb us.  */
    GLRO(dl_platform) = NULL;

#ifdef SHARED
  /* init_cpu_features has been called early from __libc_start_main in
     static executable.  */
  init_cpu_features (&GLRO(dl_aarch64_cpu_features));
#endif
}


static inline ElfW(Addr)
elf_machine_fixup_plt (struct link_map *map, lookup_t t,
		       const ElfW(Sym) *refsym, const ElfW(Sym) *sym,
		       const ElfW(Rela) *reloc,
		       ElfW(Addr) *reloc_addr,
		       ElfW(Addr) value)
{
  return *reloc_addr = value;
}

/* Return the final value of a plt relocation.  */
static inline ElfW(Addr)
elf_machine_plt_value (struct link_map *map,
		       const ElfW(Rela) *reloc,
		       ElfW(Addr) value)
{
  return value;
}

#endif

/* Names of the architecture-specific auditing callback functions.  */
#define ARCH_LA_PLTENTER aarch64_gnu_pltenter
#define ARCH_LA_PLTEXIT  aarch64_gnu_pltexit

#ifdef RESOLVE_MAP

auto inline void
__attribute__ ((always_inline))
elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc,
		  const ElfW(Sym) *sym, const struct r_found_version *version,
		  void *const reloc_addr_arg, int skip_ifunc)
{
  ElfW(Addr) *const reloc_addr = reloc_addr_arg;
  const unsigned int r_type = ELFW (R_TYPE) (reloc->r_info);

  if (__builtin_expect (r_type == AARCH64_R(RELATIVE), 0))
      *reloc_addr = map->l_addr + reloc->r_addend;
  else if (__builtin_expect (r_type == R_AARCH64_NONE, 0))
      return;
  else
    {
      const ElfW(Sym) *const refsym = sym;
      struct link_map *sym_map = RESOLVE_MAP (&sym, version, r_type);
      ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);

      if (sym != NULL
	  && __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
	  && __glibc_likely (sym->st_shndx != SHN_UNDEF)
	  && __glibc_likely (!skip_ifunc))
	value = elf_ifunc_invoke (value);

      switch (r_type)
	{
	case AARCH64_R(COPY):
	  if (sym == NULL)
	      break;

	  if (sym->st_size > refsym->st_size
	      || (GLRO(dl_verbose) && sym->st_size < refsym->st_size))
	    {
	      const char *strtab;

	      strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
	      _dl_error_printf ("\
%s: Symbol `%s' has different size in shared object, consider re-linking\n",
				RTLD_PROGNAME, strtab + refsym->st_name);
	    }
	  memcpy (reloc_addr_arg, (void *) value,
		  sym->st_size < refsym->st_size
		  ? sym->st_size : refsym->st_size);
	  break;

	case AARCH64_R(RELATIVE):
	case AARCH64_R(GLOB_DAT):
	case AARCH64_R(JUMP_SLOT):
	case AARCH64_R(ABS32):
#ifdef __LP64__
	case AARCH64_R(ABS64):
#endif
	  *reloc_addr = value + reloc->r_addend;
	  break;

	case AARCH64_R(TLSDESC):
	  {
	    struct tlsdesc volatile *td =
	      (struct tlsdesc volatile *)reloc_addr;
#ifndef RTLD_BOOTSTRAP
	    if (! sym)
	      {
		td->arg = (void*)reloc->r_addend;
		td->entry = _dl_tlsdesc_undefweak;
	      }
	    else
#endif
	      {
#ifndef RTLD_BOOTSTRAP
# ifndef SHARED
		CHECK_STATIC_TLS (map, sym_map);
# else
		if (!TRY_STATIC_TLS (map, sym_map))
		  {
		    td->arg = _dl_make_tlsdesc_dynamic
		      (sym_map, sym->st_value + reloc->r_addend);
		    td->entry = _dl_tlsdesc_dynamic;
		  }
		else
# endif
#endif
		  {
		    td->arg = (void*)(sym->st_value + sym_map->l_tls_offset
				      + reloc->r_addend);
		    td->entry = _dl_tlsdesc_return;
		  }
	      }
	    break;
	  }

	case AARCH64_R(TLS_DTPMOD):
#ifdef RTLD_BOOTSTRAP
	  *reloc_addr = 1;
#else
	  if (sym_map != NULL)
	    {
	      *reloc_addr = sym_map->l_tls_modid;
	    }
#endif
	  break;

	case AARCH64_R(TLS_DTPREL):
	  if (sym)
	    *reloc_addr = sym->st_value + reloc->r_addend;
	  break;

	case AARCH64_R(TLS_TPREL):
	  if (sym)
	    {
	      CHECK_STATIC_TLS (map, sym_map);
	      *reloc_addr =
		sym->st_value + reloc->r_addend + sym_map->l_tls_offset;
	    }
	  break;

	case AARCH64_R(IRELATIVE):
	  value = map->l_addr + reloc->r_addend;
	  value = elf_ifunc_invoke (value);
	  *reloc_addr = value;
	  break;

	default:
	  _dl_reloc_bad_type (map, r_type, 0);
	  break;
	}
    }
}

inline void
__attribute__ ((always_inline))
elf_machine_rela_relative (ElfW(Addr) l_addr,
			   const ElfW(Rela) *reloc,
			   void *const reloc_addr_arg)
{
  ElfW(Addr) *const reloc_addr = reloc_addr_arg;
  *reloc_addr = l_addr + reloc->r_addend;
}

inline void
__attribute__ ((always_inline))
elf_machine_lazy_rel (struct link_map *map,
		      ElfW(Addr) l_addr,
		      const ElfW(Rela) *reloc,
		      int skip_ifunc)
{
  ElfW(Addr) *const reloc_addr = (void *) (l_addr + reloc->r_offset);
  const unsigned int r_type = ELFW (R_TYPE) (reloc->r_info);
  /* Check for unexpected PLT reloc type.  */
  if (__builtin_expect (r_type == AARCH64_R(JUMP_SLOT), 1))
    {
      if (map->l_mach.plt == 0)
	{
	  /* Prelinking.  */
	  *reloc_addr += l_addr;
	  return;
	}

      if (__glibc_unlikely (map->l_info[DT_AARCH64 (VARIANT_PCS)] != NULL))
	{
	  /* Check the symbol table for variant PCS symbols.  */
	  const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
	  const ElfW (Sym) *symtab =
	    (const void *)D_PTR (map, l_info[DT_SYMTAB]);
	  const ElfW (Sym) *sym = &symtab[symndx];
	  if (__glibc_unlikely (sym->st_other & STO_AARCH64_VARIANT_PCS))
	    {
	      /* Avoid lazy resolution of variant PCS symbols.  */
	      const struct r_found_version *version = NULL;
	      if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
		{
		  const ElfW (Half) *vernum =
		    (const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
		  version = &map->l_versions[vernum[symndx] & 0x7fff];
		}
	      elf_machine_rela (map, reloc, sym, version, reloc_addr,
				skip_ifunc);
	      return;
	    }
	}

      *reloc_addr = map->l_mach.plt;
    }
  else if (__builtin_expect (r_type == AARCH64_R(TLSDESC), 1))
    {
      const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
      const ElfW (Sym) *symtab = (const void *)D_PTR (map, l_info[DT_SYMTAB]);
      const ElfW (Sym) *sym = &symtab[symndx];
      const struct r_found_version *version = NULL;

      if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
	{
	  const ElfW (Half) *vernum =
	    (const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
	  version = &map->l_versions[vernum[symndx] & 0x7fff];
	}

      /* Always initialize TLS descriptors completely, because lazy
	 initialization requires synchronization at every TLS access.  */
      elf_machine_rela (map, reloc, sym, version, reloc_addr, skip_ifunc);
    }
  else if (__glibc_unlikely (r_type == AARCH64_R(IRELATIVE)))
    {
      ElfW(Addr) value = map->l_addr + reloc->r_addend;
      if (__glibc_likely (!skip_ifunc))
	value = elf_ifunc_invoke (value);
      *reloc_addr = value;
    }
  else
    _dl_reloc_bad_type (map, r_type, 1);
}

#endif
Commit	Line	Data
04277e02	1	/* Copyright (C) 1995-2019 Free Software Foundation, Inc.
554066b8 MS	2
	3	This file is part of the GNU C Library.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public License as
	7	published by the Free Software Foundation; either version 2.1 of the
	8	License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
	16	License along with the GNU C Library; if not, see
5a82c748	17	<https://www.gnu.org/licenses/>. */
554066b8 MS	18
	19	#ifndef dl_machine_h
	20	#define dl_machine_h
	21
	22	#define ELF_MACHINE_NAME "aarch64"
	23
389d1f1b	24	#include <sysdep.h>
554066b8 MS	25	#include <tls.h>
554066b8 MS	26	#include <dl-tlsdesc.h>
7520ff8c	27	#include <dl-irel.h>
d2e4346a	28	#include <cpu-features.c>
554066b8	29
82bc69c0 SN	30	/* Translate a processor specific dynamic tag to the index in l_info array. */
	31	#define DT_AARCH64(x) (DT_AARCH64_##x - DT_LOPROC + DT_NUM)
	32
554066b8 MS	33	/* Return nonzero iff ELF header is compatible with the running host. */
	34	static inline int __attribute__ ((unused))
	35	elf_machine_matches_host (const ElfW(Ehdr) *ehdr)
	36	{
	37	return ehdr->e_machine == EM_AARCH64;
	38	}
	39
	40	/* Return the link-time address of _DYNAMIC. Conveniently, this is the
	41	first element of the GOT. */
	42	static inline ElfW(Addr) __attribute__ ((unused))
	43	elf_machine_dynamic (void)
	44	{
ed0257f7 MS	45	extern const ElfW(Addr) _GLOBAL_OFFSET_TABLE_[] attribute_hidden;
ed0257f7 MS	46	return _GLOBAL_OFFSET_TABLE_[0];
554066b8 MS	47	}
	48
	49	/* Return the run-time load address of the shared object. */
	50
	51	static inline ElfW(Addr) __attribute__ ((unused))
	52	elf_machine_load_address (void)
	53	{
	54	/* To figure out the load address we use the definition that for any symbol:
	55	dynamic_addr(symbol) = static_addr(symbol) + load_addr
	56
a68ba2f3 SN	57	_DYNAMIC sysmbol is used here as its link-time address stored in
a68ba2f3 SN	58	the special unrelocated first GOT entry. */
554066b8	59
a68ba2f3 SN	60	extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
a68ba2f3 SN	61	return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic ();
554066b8 MS	62	}
	63
	64	/* Set up the loaded object described by L so its unrelocated PLT
	65	entries will jump to the on-demand fixup code in dl-runtime.c. */
	66
	67	static inline int __attribute__ ((unused))
	68	elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
	69	{
	70	if (l->l_info[DT_JMPREL] && lazy)
	71	{
	72	ElfW(Addr) *got;
	73	extern void _dl_runtime_resolve (ElfW(Word));
	74	extern void _dl_runtime_profile (ElfW(Word));
	75
	76	got = (ElfW(Addr) *) D_PTR (l, l_info[DT_PLTGOT]);
	77	if (got[1])
	78	{
	79	l->l_mach.plt = got[1] + l->l_addr;
	80	}
	81	got[1] = (ElfW(Addr)) l;
	82
	83	/* The got[2] entry contains the address of a function which gets
	84	called to get the address of a so far unresolved function and
	85	jump to it. The profiling extension of the dynamic linker allows
	86	to intercept the calls to collect information. In this case we
	87	don't store the address in the GOT so that all future calls also
	88	end in this function. */
	89	if ( profile)
	90	{
	91	got[2] = (ElfW(Addr)) &_dl_runtime_profile;
	92
	93	if (GLRO(dl_profile) != NULL
	94	&& _dl_name_match_p (GLRO(dl_profile), l))
	95	/* Say that we really want profiling and the timers are
	96	started. */
	97	GL(dl_profile_map) = l;
	98	}
	99	else
	100	{
	101	/* This function will get called to fix up the GOT entry
	102	indicated by the offset on the stack, and then jump to
	103	the resolved address. */
	104	got[2] = (ElfW(Addr)) &_dl_runtime_resolve;
	105	}
	106	}
	107
554066b8 MS	108	return lazy;
	109	}
	110
	111	/* Initial entry point for the dynamic linker. The C function
	112	_dl_start is the real entry point, its return value is the user
	113	program's entry point */
389d1f1b SE	114	#ifdef __LP64__
	115	# define RTLD_START RTLD_START_1 ("x", "3", "sp")
	116	#else
	117	# define RTLD_START RTLD_START_1 ("w", "2", "wsp")
	118	#endif
554066b8	119
389d1f1b SE	120
	121	#define RTLD_START_1(PTR, PTR_SIZE_LOG, PTR_SP) asm ("\
	122	.text \n\
	123	.globl _start \n\
	124	.type _start, %function \n\
	125	.globl _dl_start_user \n\
	126	.type _dl_start_user, %function \n\
	127	_start: \n\
	128	mov " PTR "0, " PTR_SP " \n\
	129	bl _dl_start \n\
	130	// returns user entry point in x0 \n\
	131	mov x21, x0 \n\
	132	_dl_start_user: \n\
	133	// get the original arg count \n\
	134	ldr " PTR "1, [sp] \n\
	135	// get the argv address \n\
	136	add " PTR "2, " PTR_SP ", #(1<<" PTR_SIZE_LOG ") \n\
	137	// get _dl_skip_args to see if we were \n\
	138	// invoked as an executable \n\
	139	adrp x4, _dl_skip_args \n\
	140	ldr w4, [x4, #:lo12:_dl_skip_args] \n\
	141	// do we need to adjust argc/argv \n\
	142	cmp w4, 0 \n\
	143	beq .L_done_stack_adjust \n\
	144	// subtract _dl_skip_args from original arg count \n\
	145	sub " PTR "1, " PTR "1, " PTR "4 \n\
	146	// store adjusted argc back to stack \n\
	147	str " PTR "1, [sp] \n\
	148	// find the first unskipped argument \n\
	149	mov " PTR "3, " PTR "2 \n\
	150	add " PTR "4, " PTR "2, " PTR "4, lsl #" PTR_SIZE_LOG " \n\
	151	// shuffle argv down \n\
	152	1: ldr " PTR "5, [x4], #(1<<" PTR_SIZE_LOG ") \n\
	153	str " PTR "5, [x3], #(1<<" PTR_SIZE_LOG ") \n\
	154	cmp " PTR "5, #0 \n\
	155	bne 1b \n\
	156	// shuffle envp down \n\
	157	1: ldr " PTR "5, [x4], #(1<<" PTR_SIZE_LOG ") \n\
	158	str " PTR "5, [x3], #(1<<" PTR_SIZE_LOG ") \n\
	159	cmp " PTR "5, #0 \n\
	160	bne 1b \n\
	161	// shuffle auxv down \n\
	162	1: ldp " PTR "0, " PTR "5, [x4, #(2<<" PTR_SIZE_LOG ")]! \n\
	163	stp " PTR "0, " PTR "5, [x3], #(2<<" PTR_SIZE_LOG ") \n\
	164	cmp " PTR "0, #0 \n\
	165	bne 1b \n\
	166	// Update _dl_argv \n\
e9177fba SN	167	adrp x3, __GI__dl_argv \n\
e9177fba SN	168	str " PTR "2, [x3, #:lo12:__GI__dl_argv] \n\
389d1f1b SE	169	.L_done_stack_adjust: \n\
	170	// compute envp \n\
	171	add " PTR "3, " PTR "2, " PTR "1, lsl #" PTR_SIZE_LOG " \n\
	172	add " PTR "3, " PTR "3, #(1<<" PTR_SIZE_LOG ") \n\
	173	adrp x16, _rtld_local \n\
	174	add " PTR "16, " PTR "16, #:lo12:_rtld_local \n\
	175	ldr " PTR "0, [x16] \n\
	176	bl _dl_init \n\
	177	// load the finalizer function \n\
	178	adrp x0, _dl_fini \n\
	179	add " PTR "0, " PTR "0, #:lo12:_dl_fini \n\
	180	// jump to the user_s entry point \n\
	181	br x21 \n\
554066b8 MS	182	");
	183
	184	#define elf_machine_type_class(type) \
389d1f1b SE	185	((((type) == AARCH64_R(JUMP_SLOT) \
	186	\|\| (type) == AARCH64_R(TLS_DTPMOD) \
	187	\|\| (type) == AARCH64_R(TLS_DTPREL) \
	188	\|\| (type) == AARCH64_R(TLS_TPREL) \
	189	\|\| (type) == AARCH64_R(TLSDESC)) * ELF_RTYPE_CLASS_PLT) \
	190	\| (((type) == AARCH64_R(COPY)) * ELF_RTYPE_CLASS_COPY) \
	191	\| (((type) == AARCH64_R(GLOB_DAT)) * ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA))
554066b8	192
389d1f1b	193	#define ELF_MACHINE_JMP_SLOT AARCH64_R(JUMP_SLOT)
554066b8 MS	194
	195	/* AArch64 uses RELA not REL */
	196	#define ELF_MACHINE_NO_REL 1
4cf5b6d0	197	#define ELF_MACHINE_NO_RELA 0
554066b8	198
d2e4346a SE	199	#define DL_PLATFORM_INIT dl_platform_init ()
	200
	201	static inline void __attribute__ ((unused))
	202	dl_platform_init (void)
	203	{
	204	if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
	205	/* Avoid an empty string which would disturb us. */
	206	GLRO(dl_platform) = NULL;
	207
	208	#ifdef SHARED
	209	/* init_cpu_features has been called early from __libc_start_main in
	210	static executable. */
	211	init_cpu_features (&GLRO(dl_aarch64_cpu_features));
	212	#endif
	213	}
	214
	215
554066b8 MS	216	static inline ElfW(Addr)
554066b8 MS	217	elf_machine_fixup_plt (struct link_map *map, lookup_t t,
0572433b	218	const ElfW(Sym) refsym, const ElfW(Sym) sym,
554066b8 MS	219	const ElfW(Rela) *reloc,
	220	ElfW(Addr) *reloc_addr,
	221	ElfW(Addr) value)
	222	{
	223	return *reloc_addr = value;
	224	}
	225
	226	/* Return the final value of a plt relocation. */
	227	static inline ElfW(Addr)
	228	elf_machine_plt_value (struct link_map *map,
	229	const ElfW(Rela) *reloc,
	230	ElfW(Addr) value)
	231	{
	232	return value;
	233	}
	234
	235	#endif
	236
	237	/* Names of the architecture-specific auditing callback functions. */
	238	#define ARCH_LA_PLTENTER aarch64_gnu_pltenter
	239	#define ARCH_LA_PLTEXIT aarch64_gnu_pltexit
	240
	241	#ifdef RESOLVE_MAP
	242
	243	auto inline void
	244	__attribute__ ((always_inline))
	245	elf_machine_rela (struct link_map map, const ElfW(Rela) reloc,
	246	const ElfW(Sym) sym, const struct r_found_version version,
	247	void *const reloc_addr_arg, int skip_ifunc)
	248	{
	249	ElfW(Addr) *const reloc_addr = reloc_addr_arg;
389d1f1b	250	const unsigned int r_type = ELFW (R_TYPE) (reloc->r_info);
554066b8	251
389d1f1b	252	if (__builtin_expect (r_type == AARCH64_R(RELATIVE), 0))
554066b8 MS	253	*reloc_addr = map->l_addr + reloc->r_addend;
	254	else if (__builtin_expect (r_type == R_AARCH64_NONE, 0))
	255	return;
	256	else
	257	{
	258	const ElfW(Sym) *const refsym = sym;
	259	struct link_map *sym_map = RESOLVE_MAP (&sym, version, r_type);
10a446dd	260	ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);
554066b8	261
7520ff8c WN	262	if (sym != NULL
	263	&& __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
	264	&& __glibc_likely (sym->st_shndx != SHN_UNDEF)
	265	&& __glibc_likely (!skip_ifunc))
	266	value = elf_ifunc_invoke (value);
	267
554066b8 MS	268	switch (r_type)
554066b8 MS	269	{
389d1f1b	270	case AARCH64_R(COPY):
554066b8 MS	271	if (sym == NULL)
	272	break;
	273
	274	if (sym->st_size > refsym->st_size
	275	\|\| (GLRO(dl_verbose) && sym->st_size < refsym->st_size))
	276	{
	277	const char *strtab;
	278
	279	strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
	280	_dl_error_printf ("\
	281	%s: Symbol `%s' has different size in shared object, consider re-linking\n",
b9375348	282	RTLD_PROGNAME, strtab + refsym->st_name);
554066b8 MS	283	}
554066b8 MS	284	memcpy (reloc_addr_arg, (void *) value,
db4f87ba SN	285	sym->st_size < refsym->st_size
db4f87ba SN	286	? sym->st_size : refsym->st_size);
554066b8 MS	287	break;
554066b8 MS	288
389d1f1b SE	289	case AARCH64_R(RELATIVE):
	290	case AARCH64_R(GLOB_DAT):
	291	case AARCH64_R(JUMP_SLOT):
	292	case AARCH64_R(ABS32):
	293	#ifdef __LP64__
	294	case AARCH64_R(ABS64):
	295	#endif
554066b8 MS	296	*reloc_addr = value + reloc->r_addend;
	297	break;
	298
389d1f1b	299	case AARCH64_R(TLSDESC):
554066b8 MS	300	{
	301	struct tlsdesc volatile *td =
	302	(struct tlsdesc volatile *)reloc_addr;
	303	#ifndef RTLD_BOOTSTRAP
	304	if (! sym)
	305	{
	306	td->arg = (void*)reloc->r_addend;
	307	td->entry = _dl_tlsdesc_undefweak;
	308	}
	309	else
	310	#endif
	311	{
	312	#ifndef RTLD_BOOTSTRAP
	313	# ifndef SHARED
	314	CHECK_STATIC_TLS (map, sym_map);
	315	# else
	316	if (!TRY_STATIC_TLS (map, sym_map))
	317	{
	318	td->arg = _dl_make_tlsdesc_dynamic
	319	(sym_map, sym->st_value + reloc->r_addend);
	320	td->entry = _dl_tlsdesc_dynamic;
	321	}
	322	else
	323	# endif
	324	#endif
	325	{
	326	td->arg = (void*)(sym->st_value + sym_map->l_tls_offset
	327	+ reloc->r_addend);
	328	td->entry = _dl_tlsdesc_return;
	329	}
	330	}
	331	break;
	332	}
	333
389d1f1b	334	case AARCH64_R(TLS_DTPMOD):
554066b8 MS	335	#ifdef RTLD_BOOTSTRAP
	336	*reloc_addr = 1;
	337	#else
	338	if (sym_map != NULL)
	339	{
	340	*reloc_addr = sym_map->l_tls_modid;
	341	}
	342	#endif
	343	break;
	344
389d1f1b	345	case AARCH64_R(TLS_DTPREL):
554066b8	346	if (sym)
443d9489	347	*reloc_addr = sym->st_value + reloc->r_addend;
554066b8 MS	348	break;
554066b8 MS	349
389d1f1b	350	case AARCH64_R(TLS_TPREL):
554066b8 MS	351	if (sym)
554066b8 MS	352	{
554066b8 MS	353	CHECK_STATIC_TLS (map, sym_map);
	354	*reloc_addr =
	355	sym->st_value + reloc->r_addend + sym_map->l_tls_offset;
	356	}
	357	break;
	358
389d1f1b	359	case AARCH64_R(IRELATIVE):
7520ff8c WN	360	value = map->l_addr + reloc->r_addend;
	361	value = elf_ifunc_invoke (value);
	362	*reloc_addr = value;
	363	break;
	364
554066b8 MS	365	default:
	366	_dl_reloc_bad_type (map, r_type, 0);
	367	break;
	368	}
	369	}
	370	}
	371
	372	inline void
	373	__attribute__ ((always_inline))
	374	elf_machine_rela_relative (ElfW(Addr) l_addr,
	375	const ElfW(Rela) *reloc,
	376	void *const reloc_addr_arg)
	377	{
	378	ElfW(Addr) *const reloc_addr = reloc_addr_arg;
	379	*reloc_addr = l_addr + reloc->r_addend;
	380	}
	381
	382	inline void
	383	__attribute__ ((always_inline))
	384	elf_machine_lazy_rel (struct link_map *map,
	385	ElfW(Addr) l_addr,
	386	const ElfW(Rela) *reloc,
	387	int skip_ifunc)
	388	{
	389	ElfW(Addr) const reloc_addr = (void ) (l_addr + reloc->r_offset);
389d1f1b	390	const unsigned int r_type = ELFW (R_TYPE) (reloc->r_info);
554066b8	391	/* Check for unexpected PLT reloc type. */
389d1f1b	392	if (__builtin_expect (r_type == AARCH64_R(JUMP_SLOT), 1))
554066b8	393	{
82bc69c0 SN	394	if (map->l_mach.plt == 0)
	395	{
	396	/* Prelinking. */
	397	*reloc_addr += l_addr;
	398	return;
	399	}
	400
30ba0375	401	if (__glibc_unlikely (map->l_info[DT_AARCH64 (VARIANT_PCS)] != NULL))
82bc69c0 SN	402	{
	403	/* Check the symbol table for variant PCS symbols. */
	404	const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
	405	const ElfW (Sym) *symtab =
	406	(const void *)D_PTR (map, l_info[DT_SYMTAB]);
	407	const ElfW (Sym) *sym = &symtab[symndx];
	408	if (__glibc_unlikely (sym->st_other & STO_AARCH64_VARIANT_PCS))
	409	{
	410	/* Avoid lazy resolution of variant PCS symbols. */
	411	const struct r_found_version *version = NULL;
	412	if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
	413	{
	414	const ElfW (Half) *vernum =
	415	(const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
	416	version = &map->l_versions[vernum[symndx] & 0x7fff];
	417	}
	418	elf_machine_rela (map, reloc, sym, version, reloc_addr,
	419	skip_ifunc);
	420	return;
	421	}
	422	}
	423
	424	*reloc_addr = map->l_mach.plt;
554066b8	425	}
389d1f1b	426	else if (__builtin_expect (r_type == AARCH64_R(TLSDESC), 1))
554066b8	427	{
b7cf203b SN	428	const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
	429	const ElfW (Sym) symtab = (const void )D_PTR (map, l_info[DT_SYMTAB]);
	430	const ElfW (Sym) *sym = &symtab[symndx];
	431	const struct r_found_version *version = NULL;
554066b8	432
b7cf203b SN	433	if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
	434	{
	435	const ElfW (Half) *vernum =
	436	(const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
	437	version = &map->l_versions[vernum[symndx] & 0x7fff];
	438	}
	439
	440	/* Always initialize TLS descriptors completely, because lazy
	441	initialization requires synchronization at every TLS access. */
	442	elf_machine_rela (map, reloc, sym, version, reloc_addr, skip_ifunc);
554066b8	443	}
389d1f1b	444	else if (__glibc_unlikely (r_type == AARCH64_R(IRELATIVE)))
7520ff8c WN	445	{
	446	ElfW(Addr) value = map->l_addr + reloc->r_addend;
	447	if (__glibc_likely (!skip_ifunc))
	448	value = elf_ifunc_invoke (value);
	449	*reloc_addr = value;
	450	}
554066b8 MS	451	else
	452	_dl_reloc_bad_type (map, r_type, 1);
	453	}
	454
	455	#endif