[thirdparty/glibc.git] / sysdeps / powerpc / powerpc64 / sysdep.h

/* Assembly macros for 64-bit PowerPC.
   Copyright (C) 2002-2021 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdeps/powerpc/sysdep.h>
#include <tls.h>

#ifdef __ASSEMBLER__

/* Stack frame offsets.  */
#define FRAME_BACKCHAIN		0
#define FRAME_CR_SAVE		8
#define FRAME_LR_SAVE		16
#if _CALL_ELF != 2
#define FRAME_MIN_SIZE		112
#define FRAME_MIN_SIZE_PARM	112
#define FRAME_TOC_SAVE		40
#define FRAME_PARM_SAVE		48
#else
#define FRAME_MIN_SIZE		32
#define FRAME_MIN_SIZE_PARM	96
#define FRAME_TOC_SAVE		24
#define FRAME_PARM_SAVE		32
#endif

/* Support macros for CALL_MCOUNT.  */
	.macro SAVE_ARG NARG
	.if \NARG
	SAVE_ARG \NARG-1
	std	2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
	.endif
	.endm

	.macro REST_ARG NARG
	.if \NARG
	REST_ARG \NARG-1
	ld	2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
	.endif
	.endm

	.macro CFI_SAVE_ARG NARG
	.if \NARG
	CFI_SAVE_ARG \NARG-1
	cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
	.endif
	.endm

	.macro CFI_REST_ARG NARG
	.if \NARG
	CFI_REST_ARG \NARG-1
	cfi_restore(2+\NARG)
	.endif
	.endm

/* If compiled for profiling, call `_mcount' at the start of each function.
   see ppc-mcount.S for more details.  */
	.macro CALL_MCOUNT NARG
#ifdef	PROF
	mflr	r0
	SAVE_ARG \NARG
	std	r0,FRAME_LR_SAVE(r1)
	stdu	r1,-FRAME_MIN_SIZE_PARM(r1)
	cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
	cfi_offset(lr,FRAME_LR_SAVE)
	CFI_SAVE_ARG \NARG
	bl	JUMPTARGET (_mcount)
#ifndef SHARED
	nop
#endif
	ld	r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
	REST_ARG \NARG
	mtlr	r0
	addi	r1,r1,FRAME_MIN_SIZE_PARM
	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
	cfi_restore(lr)
	CFI_REST_ARG \NARG
#endif
	.endm

#if _CALL_ELF != 2

/* Macro to prepare for calling via a function pointer.  */
	.macro PPC64_LOAD_FUNCPTR PTR
	ld      r12,0(\PTR)
	ld      r2,8(\PTR)
	mtctr   r12
	ld      r11,16(\PTR)
	.endm

#ifdef USE_PPC64_OVERLAPPING_OPD
# define OPD_ENT(name)	.quad BODY_LABEL (name), .TOC.@tocbase
#else
# define OPD_ENT(name)	.quad BODY_LABEL (name), .TOC.@tocbase, 0
#endif

#define ENTRY_1(name)				\
	.type BODY_LABEL(name),@function;	\
	.globl name;				\
	.section ".opd","aw";			\
	.p2align 3;FUNC_LABEL(name):		\
	OPD_ENT (name);				\
	.previous

#define FUNC_LABEL(X) X
#define BODY_LABEL(X) .LY##X
#define ENTRY_2(name)				\
	.type name,@function;			\
	ENTRY_1(name)
#define END_2(name)				\
	.size name,.-BODY_LABEL(name);		\
	.size BODY_LABEL(name),.-BODY_LABEL(name)
#define LOCALENTRY(name)

#else /* _CALL_ELF == 2 */

/* Macro to prepare for calling via a function pointer.  */
	.macro PPC64_LOAD_FUNCPTR PTR
	mr	r12,\PTR
	mtctr   r12
	.endm

#define FUNC_LABEL(X) X
#define BODY_LABEL(X) X
#define ENTRY_2(name)				\
	.globl name;				\
	.type name,@function
#define END_2(name)				\
	.size name,.-name
#define LOCALENTRY(name)			\
1:      addis	r2,r12,.TOC.-1b@ha;		\
        addi	r2,r2,.TOC.-1b@l;		\
	.localentry name,.-name

#endif /* _CALL_ELF */

	.macro NOPS NARG
	.if \NARG
	NOPS \NARG-1
	nop
	.endif
	.endm

	.macro ENTRY_3 name, alignp2=2, nopwords=0
	.text
	ENTRY_2(\name)
	.p2align \alignp2
	NOPS \nopwords
BODY_LABEL(\name):
	.endm

/* Use ENTRY_TOCLESS for functions that make no use of r2 and
   guarantee r2 is unchanged on exit.  Any function that has @toc or
   @got relocs uses r2.  Functions that call other functions via the
   PLT use r2.  Use ENTRY for functions that may use or change r2.
   The first argument is the function name.
   The optional second argument specifies alignment of the function's
   code, as the logarithm base two of the byte alignment.  For
   example, a value of four aligns to a sixteen byte boundary.
   The optional third argument specifies the number of NOPs to emit
   before the start of the function's code.   */
#ifndef PROF
#define ENTRY_TOCLESS(name, ...)		\
	ENTRY_3 name, ## __VA_ARGS__;		\
	cfi_startproc

#define ENTRY(name, ...)			\
	ENTRY_TOCLESS(name, ## __VA_ARGS__);	\
	LOCALENTRY(name)
#else
/* The call to _mcount is potentially via the plt, so profiling code
   is never free of an r2 use.  */
#define ENTRY_TOCLESS(name, ...)		\
	ENTRY_3 name, ## __VA_ARGS__;		\
	cfi_startproc;				\
	LOCALENTRY(name)

#define ENTRY(name, ...)			\
	ENTRY_TOCLESS(name, ## __VA_ARGS__)
#endif

/* Local labels stripped out by the linker.  */
#undef L
#define L(x) .L##x

#define tostring(s) #s
#define stringify(s) tostring(s)
#define XGLUE(a,b) a##b
#define GLUE(a,b) XGLUE(a,b)
#define LT_LABEL(name) GLUE(.LT,name)
#define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)

/* Support Traceback tables */
#define TB_ASM			0x000c000000000000
#define TB_GLOBALLINK		0x0000800000000000
#define TB_IS_EPROL		0x0000400000000000
#define TB_HAS_TBOFF		0x0000200000000000
#define TB_INT_PROC		0x0000100000000000
#define TB_HAS_CTL		0x0000080000000000
#define TB_TOCLESS		0x0000040000000000
#define TB_FP_PRESENT		0x0000020000000000
#define TB_LOG_ABORT		0x0000010000000000
#define TB_INT_HANDL		0x0000008000000000
#define TB_NAME_PRESENT		0x0000004000000000
#define TB_USES_ALLOCA		0x0000002000000000
#define TB_SAVES_CR		0x0000000200000000
#define TB_SAVES_LR		0x0000000100000000
#define TB_STORES_BC		0x0000000080000000
#define TB_FIXUP		0x0000000040000000
#define TB_FP_SAVED(fprs)	(((fprs) & 0x3f) << 24)
#define TB_GPR_SAVED(gprs)	(((fprs) & 0x3f) << 16)
#define TB_FIXEDPARMS(parms)	(((parms) & 0xff) << 8)
#define TB_FLOATPARMS(parms)	(((parms) & 0x7f) << 1)
#define TB_PARMSONSTK		0x0000000000000001

#define PPC_HIGHER(v) 		(((v) >> 32) & 0xffff)
#define TB_DEFAULT		TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT

#define TRACEBACK(name) \
LT_LABEL(name): ; \
	.long	0 ; \
	.quad	TB_DEFAULT ; \
	.long	LT_LABEL(name)-BODY_LABEL(name) ; \
	.short	LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
LT_LABELSUFFIX(name,_name_start): ;\
	.ascii	stringify(name) ; \
LT_LABELSUFFIX(name,_name_end): ; \
	.p2align 2

#define TRACEBACK_MASK(name,mask) \
LT_LABEL(name): ; \
	.long	0 ; \
	.quad	TB_DEFAULT | mask ; \
	.long	LT_LABEL(name)-BODY_LABEL(name) ; \
	.short	LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
LT_LABELSUFFIX(name,_name_start): ;\
	.ascii	stringify(name) ; \
LT_LABELSUFFIX(name,_name_end): ; \
	.p2align 2

/* END generates Traceback tables */
#undef	END
#define END(name) \
  cfi_endproc;			\
  TRACEBACK(name);		\
  END_2(name)

/* This form supports more informative traceback tables */
#define END_GEN_TB(name,mask)	\
  cfi_endproc;			\
  TRACEBACK_MASK(name,mask);	\
  END_2(name)

/* We will allocate a new frame to save LR and the non-volatile register used to
   read the TCB when checking for scv support on syscall code.  We actually just
   need the minimum frame size plus room for 1 reg (8 bytes).  But the ABI
   mandates stack frames should be aligned at 16 Bytes, so we end up allocating
   a bit more space then what will actually be used.  */
#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE

/* Allocate frame and save register */
#define NVOLREG_SAVE \
    stdu r1,-SCV_FRAME_SIZE(r1); \
    std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
    cfi_adjust_cfa_offset(SCV_FRAME_SIZE);

/* Restore register and destroy frame */
#define NVOLREG_RESTORE	\
    ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
    addi r1,r1,SCV_FRAME_SIZE; \
    cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);

/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB.  If it is not set, scv is
   not available, then go to JUMPFALSE (label given by the macro's caller).  We
   save the value we read from the TCB in a non-volatile register so we can
   reuse it later when exiting from the syscall in PSEUDO_RET.  Note that for
   the static case we need an extra check to guarantee the thread pointer has
   already been initialized, otherwise we may try to access an invalid address
   if a syscall is called before the TLS has been setup.  */
    .macro CHECK_SCV_SUPPORT REG JUMPFALSE

#ifndef SHARED
    /* Check if thread pointer has already been setup.  */
    cmpdi r13,0
    beq \JUMPFALSE
#endif

    /* Read PPC_FEATURE2_SCV from TCB and store it in REG */
    ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
    andis. \REG,\REG,PPC_FEATURE2_SCV>>16

    beq \JUMPFALSE
    .endm

#if IS_IN(rtld)
# define DO_CALL(syscall) \
    li r0,syscall; \
    DO_CALL_SC
#else
/* Before doing the syscall, check if we can use scv.  scv is supported by P9
   and later with Linux v5.9 and later.  If so, use it.  Otherwise, fallback to
   sc.  We use a non-volatile register to save hwcap2 from the TCB, so we need
   to save its content beforehand.  */
# define DO_CALL(syscall) \
    li r0,syscall; \
    NVOLREG_SAVE; \
    CHECK_SCV_SUPPORT r31 0f; \
    DO_CALL_SCV; \
    b 1f; \
0:  DO_CALL_SC; \
1:
#endif /* IS_IN(rtld) */

/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0.  */
#define DO_CALL_SC \
    sc

#define DO_CALL_SCV \
    mflr r9; \
    std r9,FRAME_LR_SAVE(r1); \
    cfi_offset(lr,FRAME_LR_SAVE); \
    scv 0; \
    ld r9,FRAME_LR_SAVE(r1); \
    mtlr r9; \
    cfi_restore(lr);

/* ppc64 is always PIC */
#undef JUMPTARGET
#define JUMPTARGET(name) FUNC_LABEL(name)

#define PSEUDO(name, syscall_name, args) \
  .section ".text";				\
  ENTRY (name);					\
  DO_CALL (SYS_ify (syscall_name))

#ifdef SHARED
#define TAIL_CALL_SYSCALL_ERROR \
    b JUMPTARGET (NOTOC (__syscall_error))
#else
/* Static version might be linked into a large app with a toc exceeding
   64k.  We can't put a toc adjusting stub on a plain branch, so can't
   tail call __syscall_error.  */
#define TAIL_CALL_SYSCALL_ERROR \
    .ifdef .Local_syscall_error; \
    b .Local_syscall_error; \
    .else; \
.Local_syscall_error: \
    mflr 0; \
    std 0,FRAME_LR_SAVE(1); \
    stdu 1,-FRAME_MIN_SIZE(1); \
    cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
    cfi_offset(lr,FRAME_LR_SAVE); \
    bl JUMPTARGET(__syscall_error); \
    nop; \
    ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
    addi 1,1,FRAME_MIN_SIZE; \
    cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
    mtlr 0; \
    cfi_restore(lr); \
    blr; \
    .endif
#endif

#if IS_IN(rtld)
# define PSEUDO_RET \
    RET_SC; \
    TAIL_CALL_SYSCALL_ERROR
#else
/* This should only be called after a DO_CALL.  In such cases, r31 contains the
   value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT.  If it is
   set, we know we have entered the kernel using scv, so handle the return code
   accordingly.  */
# define PSEUDO_RET \
    cmpdi cr5,r31,0; \
    NVOLREG_RESTORE; \
    beq cr5,0f; \
    RET_SCV; \
    b 1f; \
0:  RET_SC; \
1:  TAIL_CALL_SYSCALL_ERROR
#endif

#define RET_SCV \
    cmpdi r3,0; \
    bgelr+; \
    neg r3,r3;

#define RET_SC \
    bnslr+;

#define ret PSEUDO_RET

#undef	PSEUDO_END
#define	PSEUDO_END(name) \
  END (name)

#define PSEUDO_NOERRNO(name, syscall_name, args) \
  .section ".text";					\
  ENTRY (name);						\
  DO_CALL (SYS_ify (syscall_name))

#if IS_IN(rtld)
# define PSEUDO_RET_NOERRNO \
    blr
#else
/* This should only be called after a DO_CALL.  */
# define PSEUDO_RET_NOERRNO \
    NVOLREG_RESTORE; \
    blr
#endif /* IS_IN(rtld) */

#define ret_NOERRNO PSEUDO_RET_NOERRNO

#undef	PSEUDO_END_NOERRNO
#define	PSEUDO_END_NOERRNO(name) \
  END (name)

#define PSEUDO_ERRVAL(name, syscall_name, args) \
  .section ".text";					\
  ENTRY (name);						\
  DO_CALL (SYS_ify (syscall_name))

#if IS_IN(rtld)
# define PSEUDO_RET_ERRVAL \
    blr
#else
/* This should only be called after a DO_CALL.  */
# define PSEUDO_RET_ERRVAL \
    NVOLREG_RESTORE; \
    blr
#endif /* IS_IN(rtld) */

#define ret_ERRVAL PSEUDO_RET_ERRVAL

#undef	PSEUDO_END_ERRVAL
#define	PSEUDO_END_ERRVAL(name) \
  END (name)

#ifdef SHARED
# if IS_IN (rtld)
	 /* Inside ld.so we use the local alias to avoid runtime GOT
	    relocations.  */
#  define __GLRO_DEF(var)				\
.LC__ ## var:						\
	.tc _rtld_local_ro[TC],_rtld_local_ro
# else
#  define __GLRO_DEF(var)				\
.LC__ ## var:						\
	.tc _rtld_global_ro[TC],_rtld_global_ro
# endif
# define __GLRO(rOUT, var, offset)		\
	ld	rOUT,.LC__ ## var@toc(r2);	\
	lwz	rOUT,offset(rOUT)
#else
# define __GLRO_DEF(var)			\
.LC__ ## var:					\
	.tc _ ## var[TC],_ ## var
# define __GLRO(rOUT, var, offset)		\
	ld	rOUT,.LC__ ## var@toc(r2);	\
	lwz	rOUT,0(rOUT)
#endif

#ifdef USE_PPC64_NOTOC
# define NOTOC(l) l@notoc
#else
# define NOTOC(l) l
#endif

#else /* !__ASSEMBLER__ */

#if _CALL_ELF != 2

#define PPC64_LOAD_FUNCPTR(ptr) \
	"ld 	12,0(" #ptr ")\n"					\
	"ld	2,8(" #ptr ")\n"					\
	"mtctr	12\n"							\
	"ld	11,16(" #ptr ")"

#ifdef USE_PPC64_OVERLAPPING_OPD
# define OPD_ENT(name)	".quad " BODY_PREFIX #name ", .TOC.@tocbase"
#else
# define OPD_ENT(name)	".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
#endif

#define ENTRY_1(name)	\
	".type   " BODY_PREFIX #name ",@function\n"			\
	".globl " #name "\n"						\
	".pushsection \".opd\",\"aw\"\n"				\
	".p2align 3\n"							\
#name ":\n"								\
	OPD_ENT (name) "\n"						\
	".popsection"

#define DOT_PREFIX ""
#define BODY_PREFIX ".LY"
#define ENTRY_2(name)	\
	".type " #name ",@function\n"					\
	ENTRY_1(name)
#define END_2(name)	\
	".size " #name ",.-" BODY_PREFIX #name "\n"			\
	".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
#define LOCALENTRY(name)

#else /* _CALL_ELF */

#define PPC64_LOAD_FUNCPTR(ptr) \
	"mr	12," #ptr "\n"						\
	"mtctr 	12"

#define DOT_PREFIX ""
#define BODY_PREFIX ""
#define ENTRY_2(name)	\
	".type " #name ",@function\n"					\
	".globl " #name
#define END_2(name)	\
	".size " #name ",.-" #name
#define LOCALENTRY(name)	\
	"1: addis 2,12,.TOC.-1b@ha\n"					\
	"addi	2,2,.TOC.-1b@l\n"					\
	".localentry " #name ",.-" #name

#endif /* _CALL_ELF */

#endif	/* __ASSEMBLER__ */
Commit	Line	Data
b80a3db0	1	/* Assembly macros for 64-bit PowerPC.
2b778ceb	2	Copyright (C) 2002-2021 Free Software Foundation, Inc.
b80a3db0 RM	3	This file is part of the GNU C Library.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6	16	License along with the GNU C Library; if not, see
5a82c748	17	<https://www.gnu.org/licenses/>. */
b80a3db0	18
b80a3db0	19	#include <sysdeps/powerpc/sysdep.h>
68ab82f5	20	#include <tls.h>
b80a3db0	21
afe01786 UD	22	#ifdef __ASSEMBLER__
afe01786 UD	23
8b8a692c	24	/* Stack frame offsets. */
8b8a692c UW	25	#define FRAME_BACKCHAIN 0
	26	#define FRAME_CR_SAVE 8
	27	#define FRAME_LR_SAVE 16
fb499eb0 AM	28	#if _CALL_ELF != 2
	29	#define FRAME_MIN_SIZE 112
	30	#define FRAME_MIN_SIZE_PARM 112
8b8a692c UW	31	#define FRAME_TOC_SAVE 40
8b8a692c UW	32	#define FRAME_PARM_SAVE 48
8b8a692c UW	33	#else
	34	#define FRAME_MIN_SIZE 32
	35	#define FRAME_MIN_SIZE_PARM 96
8b8a692c UW	36	#define FRAME_TOC_SAVE 24
8b8a692c UW	37	#define FRAME_PARM_SAVE 32
8b8a692c UW	38	#endif
8b8a692c UW	39
d7d06f79 UD	40	/* Support macros for CALL_MCOUNT. */
	41	.macro SAVE_ARG NARG
	42	.if \NARG
	43	SAVE_ARG \NARG-1
dd885436	44	std 2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
d7d06f79 UD	45	.endif
	46	.endm
	47
	48	.macro REST_ARG NARG
	49	.if \NARG
	50	REST_ARG \NARG-1
dd885436	51	ld 2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
bebff237 AM	52	.endif
	53	.endm
	54
	55	.macro CFI_SAVE_ARG NARG
	56	.if \NARG
	57	CFI_SAVE_ARG \NARG-1
dd885436	58	cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
bebff237 AM	59	.endif
	60	.endm
	61
	62	.macro CFI_REST_ARG NARG
	63	.if \NARG
	64	CFI_REST_ARG \NARG-1
	65	cfi_restore(2+\NARG)
d7d06f79 UD	66	.endif
	67	.endm
	68
b80a3db0 RM	69	/* If compiled for profiling, call `_mcount' at the start of each function.
b80a3db0 RM	70	see ppc-mcount.S for more details. */
d7d06f79	71	.macro CALL_MCOUNT NARG
b80a3db0	72	#ifdef PROF
d7d06f79 UD	73	mflr r0
d7d06f79 UD	74	SAVE_ARG \NARG
8b8a692c UW	75	std r0,FRAME_LR_SAVE(r1)
	76	stdu r1,-FRAME_MIN_SIZE_PARM(r1)
	77	cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
	78	cfi_offset(lr,FRAME_LR_SAVE)
bebff237	79	CFI_SAVE_ARG \NARG
d7d06f79	80	bl JUMPTARGET (_mcount)
bebff237 AM	81	#ifndef SHARED
	82	nop
	83	#endif
8b8a692c	84	ld r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
d7d06f79	85	REST_ARG \NARG
d7d06f79	86	mtlr r0
8b8a692c UW	87	addi r1,r1,FRAME_MIN_SIZE_PARM
8b8a692c UW	88	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
bebff237 AM	89	cfi_restore(lr)
bebff237 AM	90	CFI_REST_ARG \NARG
d7d06f79 UD	91	#endif
d7d06f79 UD	92	.endm
b80a3db0	93
696caf1d UW	94	#if _CALL_ELF != 2
696caf1d UW	95
d31beafa UW	96	/* Macro to prepare for calling via a function pointer. */
	97	.macro PPC64_LOAD_FUNCPTR PTR
	98	ld r12,0(\PTR)
	99	ld r2,8(\PTR)
	100	mtctr r12
	101	ld r11,16(\PTR)
	102	.endm
	103
590b40f7 UD	104	#ifdef USE_PPC64_OVERLAPPING_OPD
	105	# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase
	106	#else
	107	# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase, 0
	108	#endif
b80a3db0	109
4b6e7667	110	#define ENTRY_1(name) \
590b40f7 UD	111	.type BODY_LABEL(name),@function; \
	112	.globl name; \
	113	.section ".opd","aw"; \
4b6e7667 AM	114	.p2align 3;FUNC_LABEL(name): \
	115	OPD_ENT (name); \
	116	.previous
590b40f7	117
4b6e7667	118	#define FUNC_LABEL(X) X
5ca10a0c	119	#define BODY_LABEL(X) .LY##X
4b6e7667	120	#define ENTRY_2(name) \
590b40f7 UD	121	.type name,@function; \
590b40f7 UD	122	ENTRY_1(name)
4b6e7667	123	#define END_2(name) \
590b40f7	124	.size name,.-BODY_LABEL(name); \
4b6e7667	125	.size BODY_LABEL(name),.-BODY_LABEL(name)
696caf1d UW	126	#define LOCALENTRY(name)
696caf1d UW	127
4b6e7667	128	#else /* _CALL_ELF == 2 */
696caf1d UW	129
	130	/* Macro to prepare for calling via a function pointer. */
	131	.macro PPC64_LOAD_FUNCPTR PTR
	132	mr r12,\PTR
	133	mtctr r12
	134	.endm
	135
4b6e7667	136	#define FUNC_LABEL(X) X
696caf1d	137	#define BODY_LABEL(X) X
4b6e7667	138	#define ENTRY_2(name) \
696caf1d	139	.globl name; \
4b6e7667 AM	140	.type name,@function
	141	#define END_2(name) \
	142	.size name,.-name
	143	#define LOCALENTRY(name) \
	144	1: addis r2,r12,.TOC.-1b@ha; \
	145	addi r2,r2,.TOC.-1b@l; \
	146	.localentry name,.-name
696caf1d UW	147
696caf1d UW	148	#endif /* _CALL_ELF */
b80a3db0	149
d5b41185 AM	150	.macro NOPS NARG
	151	.if \NARG
	152	NOPS \NARG-1
	153	nop
	154	.endif
	155	.endm
b80a3db0	156
d5b41185 AM	157	.macro ENTRY_3 name, alignp2=2, nopwords=0
	158	.text
	159	ENTRY_2(\name)
	160	.p2align \alignp2
	161	NOPS \nopwords
	162	BODY_LABEL(\name):
	163	.endm
	164
	165	/* Use ENTRY_TOCLESS for functions that make no use of r2 and
	166	guarantee r2 is unchanged on exit. Any function that has @toc or
	167	@got relocs uses r2. Functions that call other functions via the
	168	PLT use r2. Use ENTRY for functions that may use or change r2.
	169	The first argument is the function name.
	170	The optional second argument specifies alignment of the function's
	171	code, as the logarithm base two of the byte alignment. For
	172	example, a value of four aligns to a sixteen byte boundary.
	173	The optional third argument specifies the number of NOPs to emit
	174	before the start of the function's code. */
	175	#ifndef PROF
	176	#define ENTRY_TOCLESS(name, ...) \
	177	ENTRY_3 name, ## __VA_ARGS__; \
	178	cfi_startproc
	179
	180	#define ENTRY(name, ...) \
	181	ENTRY_TOCLESS(name, ## __VA_ARGS__); \
	182	LOCALENTRY(name)
	183	#else
	184	/* The call to _mcount is potentially via the plt, so profiling code
	185	is never free of an r2 use. */
	186	#define ENTRY_TOCLESS(name, ...) \
	187	ENTRY_3 name, ## __VA_ARGS__; \
696caf1d UW	188	cfi_startproc; \
696caf1d UW	189	LOCALENTRY(name)
b80a3db0	190
d5b41185 AM	191	#define ENTRY(name, ...) \
	192	ENTRY_TOCLESS(name, ## __VA_ARGS__)
	193	#endif
	194
b80a3db0 RM	195	/* Local labels stripped out by the linker. */
	196	#undef L
	197	#define L(x) .L##x
	198
	199	#define tostring(s) #s
	200	#define stringify(s) tostring(s)
	201	#define XGLUE(a,b) a##b
	202	#define GLUE(a,b) XGLUE(a,b)
	203	#define LT_LABEL(name) GLUE(.LT,name)
	204	#define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)
	205
	206	/* Support Traceback tables */
	207	#define TB_ASM 0x000c000000000000
590b40f7	208	#define TB_GLOBALLINK 0x0000800000000000
b80a3db0	209	#define TB_IS_EPROL 0x0000400000000000
590b40f7	210	#define TB_HAS_TBOFF 0x0000200000000000
b80a3db0 RM	211	#define TB_INT_PROC 0x0000100000000000
	212	#define TB_HAS_CTL 0x0000080000000000
	213	#define TB_TOCLESS 0x0000040000000000
590b40f7 UD	214	#define TB_FP_PRESENT 0x0000020000000000
	215	#define TB_LOG_ABORT 0x0000010000000000
	216	#define TB_INT_HANDL 0x0000008000000000
	217	#define TB_NAME_PRESENT 0x0000004000000000
	218	#define TB_USES_ALLOCA 0x0000002000000000
b80a3db0 RM	219	#define TB_SAVES_CR 0x0000000200000000
b80a3db0 RM	220	#define TB_SAVES_LR 0x0000000100000000
590b40f7	221	#define TB_STORES_BC 0x0000000080000000
b80a3db0 RM	222	#define TB_FIXUP 0x0000000040000000
	223	#define TB_FP_SAVED(fprs) (((fprs) & 0x3f) << 24)
	224	#define TB_GPR_SAVED(gprs) (((fprs) & 0x3f) << 16)
	225	#define TB_FIXEDPARMS(parms) (((parms) & 0xff) << 8)
	226	#define TB_FLOATPARMS(parms) (((parms) & 0x7f) << 1)
590b40f7	227	#define TB_PARMSONSTK 0x0000000000000001
b80a3db0	228
590b40f7 UD	229	#define PPC_HIGHER(v) (((v) >> 32) & 0xffff)
590b40f7 UD	230	#define TB_DEFAULT TB_ASM \| TB_HAS_TBOFF \| TB_NAME_PRESENT
b80a3db0 RM	231
	232	#define TRACEBACK(name) \
	233	LT_LABEL(name): ; \
	234	.long 0 ; \
	235	.quad TB_DEFAULT ; \
590b40f7	236	.long LT_LABEL(name)-BODY_LABEL(name) ; \
b80a3db0 RM	237	.short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
	238	LT_LABELSUFFIX(name,_name_start): ;\
	239	.ascii stringify(name) ; \
	240	LT_LABELSUFFIX(name,_name_end): ; \
4b6e7667	241	.p2align 2
b80a3db0 RM	242
	243	#define TRACEBACK_MASK(name,mask) \
	244	LT_LABEL(name): ; \
	245	.long 0 ; \
	246	.quad TB_DEFAULT \| mask ; \
590b40f7	247	.long LT_LABEL(name)-BODY_LABEL(name) ; \
b80a3db0 RM	248	.short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
	249	LT_LABELSUFFIX(name,_name_start): ;\
	250	.ascii stringify(name) ; \
	251	LT_LABELSUFFIX(name,_name_end): ; \
4b6e7667	252	.p2align 2
b80a3db0 RM	253
	254	/* END generates Traceback tables */
	255	#undef END
	256	#define END(name) \
9759bbf1	257	cfi_endproc; \
4b6e7667	258	TRACEBACK(name); \
590b40f7	259	END_2(name)
b80a3db0 RM	260
	261	/* This form supports more informative traceback tables */
	262	#define END_GEN_TB(name,mask) \
9759bbf1	263	cfi_endproc; \
4b6e7667	264	TRACEBACK_MASK(name,mask); \
590b40f7	265	END_2(name)
b80a3db0	266
68ab82f5 MC	267	/* We will allocate a new frame to save LR and the non-volatile register used to
	268	read the TCB when checking for scv support on syscall code. We actually just
	269	need the minimum frame size plus room for 1 reg (8 bytes). But the ABI
	270	mandates stack frames should be aligned at 16 Bytes, so we end up allocating
	271	a bit more space then what will actually be used. */
	272	#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
	273	#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE
	274
	275	/* Allocate frame and save register */
	276	#define NVOLREG_SAVE \
	277	stdu r1,-SCV_FRAME_SIZE(r1); \
	278	std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
	279	cfi_adjust_cfa_offset(SCV_FRAME_SIZE);
	280
	281	/* Restore register and destroy frame */
	282	#define NVOLREG_RESTORE \
	283	ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
	284	addi r1,r1,SCV_FRAME_SIZE; \
	285	cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);
	286
	287	/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is
	288	not available, then go to JUMPFALSE (label given by the macro's caller). We
	289	save the value we read from the TCB in a non-volatile register so we can
	290	reuse it later when exiting from the syscall in PSEUDO_RET. Note that for
	291	the static case we need an extra check to guarantee the thread pointer has
	292	already been initialized, otherwise we may try to access an invalid address
	293	if a syscall is called before the TLS has been setup. */
	294	.macro CHECK_SCV_SUPPORT REG JUMPFALSE
	295
	296	#ifndef SHARED
	297	/* Check if thread pointer has already been setup. */
	298	cmpdi r13,0
	299	beq \JUMPFALSE
	300	#endif
	301
	302	/* Read PPC_FEATURE2_SCV from TCB and store it in REG */
	303	ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
	304	andis. \REG,\REG,PPC_FEATURE2_SCV>>16
	305
	306	beq \JUMPFALSE
	307	.endm
	308
	309	#if IS_IN(rtld)
	310	# define DO_CALL(syscall) \
	311	li r0,syscall; \
	312	DO_CALL_SC
	313	#else
	314	/* Before doing the syscall, check if we can use scv. scv is supported by P9
	315	and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to
	316	sc. We use a non-volatile register to save hwcap2 from the TCB, so we need
	317	to save its content beforehand. */
	318	# define DO_CALL(syscall) \
	319	li r0,syscall; \
	320	NVOLREG_SAVE; \
	321	CHECK_SCV_SUPPORT r31 0f; \
	322	DO_CALL_SCV; \
	323	b 1f; \
	324	0: DO_CALL_SC; \
	325	1:
	326	#endif /* IS_IN(rtld) */
	327
	328	/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */
	329	#define DO_CALL_SC \
b80a3db0 RM	330	sc
b80a3db0 RM	331
68ab82f5 MC	332	#define DO_CALL_SCV \
	333	mflr r9; \
	334	std r9,FRAME_LR_SAVE(r1); \
	335	cfi_offset(lr,FRAME_LR_SAVE); \
	336	scv 0; \
	337	ld r9,FRAME_LR_SAVE(r1); \
	338	mtlr r9; \
	339	cfi_restore(lr);
	340
b80a3db0	341	/* ppc64 is always PIC */
29d9a17d	342	#undef JUMPTARGET
4b6e7667	343	#define JUMPTARGET(name) FUNC_LABEL(name)
b80a3db0 RM	344
b80a3db0 RM	345	#define PSEUDO(name, syscall_name, args) \
4b6e7667 AM	346	.section ".text"; \
	347	ENTRY (name); \
	348	DO_CALL (SYS_ify (syscall_name))
b80a3db0	349
bebff237 AM	350	#ifdef SHARED
bebff237 AM	351	#define TAIL_CALL_SYSCALL_ERROR \
7c7bcf36	352	b JUMPTARGET (NOTOC (__syscall_error))
bebff237 AM	353	#else
	354	/* Static version might be linked into a large app with a toc exceeding
	355	64k. We can't put a toc adjusting stub on a plain branch, so can't
	356	tail call __syscall_error. */
	357	#define TAIL_CALL_SYSCALL_ERROR \
	358	.ifdef .Local_syscall_error; \
	359	b .Local_syscall_error; \
	360	.else; \
	361	.Local_syscall_error: \
	362	mflr 0; \
8b8a692c UW	363	std 0,FRAME_LR_SAVE(1); \
	364	stdu 1,-FRAME_MIN_SIZE(1); \
	365	cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
	366	cfi_offset(lr,FRAME_LR_SAVE); \
bebff237 AM	367	bl JUMPTARGET(__syscall_error); \
bebff237 AM	368	nop; \
8b8a692c UW	369	ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
	370	addi 1,1,FRAME_MIN_SIZE; \
	371	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
bebff237 AM	372	mtlr 0; \
	373	cfi_restore(lr); \
	374	blr; \
	375	.endif
	376	#endif
	377
68ab82f5 MC	378	#if IS_IN(rtld)
	379	# define PSEUDO_RET \
	380	RET_SC; \
bebff237	381	TAIL_CALL_SYSCALL_ERROR
68ab82f5 MC	382	#else
	383	/* This should only be called after a DO_CALL. In such cases, r31 contains the
	384	value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is
	385	set, we know we have entered the kernel using scv, so handle the return code
	386	accordingly. */
	387	# define PSEUDO_RET \
	388	cmpdi cr5,r31,0; \
	389	NVOLREG_RESTORE; \
	390	beq cr5,0f; \
	391	RET_SCV; \
	392	b 1f; \
	393	0: RET_SC; \
	394	1: TAIL_CALL_SYSCALL_ERROR
	395	#endif
	396
	397	#define RET_SCV \
	398	cmpdi r3,0; \
	399	bgelr+; \
	400	neg r3,r3;
	401
	402	#define RET_SC \
	403	bnslr+;
b80a3db0 RM	404
	405	#define ret PSEUDO_RET
	406
	407	#undef PSEUDO_END
	408	#define PSEUDO_END(name) \
	409	END (name)
	410
9eb88290	411	#define PSEUDO_NOERRNO(name, syscall_name, args) \
4b6e7667 AM	412	.section ".text"; \
	413	ENTRY (name); \
	414	DO_CALL (SYS_ify (syscall_name))
9eb88290	415
68ab82f5 MC	416	#if IS_IN(rtld)
68ab82f5 MC	417	# define PSEUDO_RET_NOERRNO \
9eb88290	418	blr
68ab82f5 MC	419	#else
	420	/* This should only be called after a DO_CALL. */
	421	# define PSEUDO_RET_NOERRNO \
	422	NVOLREG_RESTORE; \
	423	blr
	424	#endif /* IS_IN(rtld) */
9eb88290 UD	425
	426	#define ret_NOERRNO PSEUDO_RET_NOERRNO
	427
	428	#undef PSEUDO_END_NOERRNO
c6289757	429	#define PSEUDO_END_NOERRNO(name) \
9eb88290 UD	430	END (name)
9eb88290 UD	431
137ffcdc	432	#define PSEUDO_ERRVAL(name, syscall_name, args) \
4b6e7667 AM	433	.section ".text"; \
	434	ENTRY (name); \
	435	DO_CALL (SYS_ify (syscall_name))
137ffcdc	436
68ab82f5 MC	437	#if IS_IN(rtld)
	438	# define PSEUDO_RET_ERRVAL \
	439	blr
	440	#else
	441	/* This should only be called after a DO_CALL. */
	442	# define PSEUDO_RET_ERRVAL \
	443	NVOLREG_RESTORE; \
137ffcdc	444	blr
68ab82f5	445	#endif /* IS_IN(rtld) */
137ffcdc UD	446
	447	#define ret_ERRVAL PSEUDO_RET_ERRVAL
	448
	449	#undef PSEUDO_END_ERRVAL
	450	#define PSEUDO_END_ERRVAL(name) \
	451	END (name)
	452
18363b4f TMQMF	453	#ifdef SHARED
	454	# if IS_IN (rtld)
	455	/* Inside ld.so we use the local alias to avoid runtime GOT
	456	relocations. */
	457	# define __GLRO_DEF(var) \
	458	.LC__ ## var: \
	459	.tc _rtld_local_ro[TC],_rtld_local_ro
	460	# else
	461	# define __GLRO_DEF(var) \
	462	.LC__ ## var: \
	463	.tc _rtld_global_ro[TC],_rtld_global_ro
	464	# endif
	465	# define __GLRO(rOUT, var, offset) \
	466	ld rOUT,.LC__ ## var@toc(r2); \
	467	lwz rOUT,offset(rOUT)
	468	#else
	469	# define __GLRO_DEF(var) \
	470	.LC__ ## var: \
	471	.tc _ ## var[TC],_ ## var
	472	# define __GLRO(rOUT, var, offset) \
	473	ld rOUT,.LC__ ## var@toc(r2); \
	474	lwz rOUT,0(rOUT)
	475	#endif
	476
7c7bcf36 TMQMF	477	#ifdef USE_PPC64_NOTOC
	478	# define NOTOC(l) l@notoc
	479	#else
	480	# define NOTOC(l) l
	481	#endif
	482
590b40f7	483	#else /* !__ASSEMBLER__ */
b80a3db0	484
696caf1d UW	485	#if _CALL_ELF != 2
696caf1d UW	486
d31beafa	487	#define PPC64_LOAD_FUNCPTR(ptr) \
4b6e7667 AM	488	"ld 12,0(" #ptr ")\n" \
	489	"ld 2,8(" #ptr ")\n" \
	490	"mtctr 12\n" \
	491	"ld 11,16(" #ptr ")"
d31beafa	492
590b40f7	493	#ifdef USE_PPC64_OVERLAPPING_OPD
4b6e7667	494	# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase"
590b40f7	495	#else
4b6e7667	496	# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
590b40f7 UD	497	#endif
590b40f7 UD	498
d31beafa	499	#define ENTRY_1(name) \
4b6e7667 AM	500	".type " BODY_PREFIX #name ",@function\n" \
	501	".globl " #name "\n" \
	502	".pushsection \".opd\",\"aw\"\n" \
	503	".p2align 3\n" \
d31beafa UW	504	#name ":\n" \
d31beafa UW	505	OPD_ENT (name) "\n" \
4b6e7667	506	".popsection"
d31beafa	507
5ca10a0c AZ	508	#define DOT_PREFIX ""
	509	#define BODY_PREFIX ".LY"
	510	#define ENTRY_2(name) \
4b6e7667	511	".type " #name ",@function\n" \
d31beafa	512	ENTRY_1(name)
5ca10a0c	513	#define END_2(name) \
4b6e7667 AM	514	".size " #name ",.-" BODY_PREFIX #name "\n" \
4b6e7667 AM	515	".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
696caf1d UW	516	#define LOCALENTRY(name)
	517
	518	#else /* _CALL_ELF */
	519
	520	#define PPC64_LOAD_FUNCPTR(ptr) \
4b6e7667 AM	521	"mr 12," #ptr "\n" \
4b6e7667 AM	522	"mtctr 12"
696caf1d UW	523
	524	#define DOT_PREFIX ""
	525	#define BODY_PREFIX ""
	526	#define ENTRY_2(name) \
4b6e7667 AM	527	".type " #name ",@function\n" \
4b6e7667 AM	528	".globl " #name
696caf1d	529	#define END_2(name) \
4b6e7667	530	".size " #name ",.-" #name
696caf1d	531	#define LOCALENTRY(name) \
4b6e7667 AM	532	"1: addis 2,12,.TOC.-1b@ha\n" \
	533	"addi 2,2,.TOC.-1b@l\n" \
	534	".localentry " #name ",.-" #name
696caf1d UW	535
696caf1d UW	536	#endif /* _CALL_ELF */
b80a3db0 RM	537
b80a3db0 RM	538	#endif /* __ASSEMBLER__ */