[thirdparty/gcc.git] / libffi / src / aarch64 / sysv.S

/* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
``Software''), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */

#define LIBFFI_ASM
#include <fficonfig.h>
#include <ffi.h>
#include <ffi_cfi.h>
#include "internal.h"

#ifdef HAVE_MACHINE_ASM_H
#include <machine/asm.h>
#else
#ifdef __USER_LABEL_PREFIX__
#define CONCAT1(a, b) CONCAT2(a, b)
#define CONCAT2(a, b) a ## b

/* Use the right prefix for global labels.  */
#define CNAME(x) CONCAT1 (__USER_LABEL_PREFIX__, x)
#else
#define CNAME(x) x
#endif
#endif

#ifdef __AARCH64EB__
# define BE(X)	X
#else
# define BE(X)	0
#endif

	.text
	.align 4

/* ffi_call_SYSV
   extern void ffi_call_SYSV (void *stack, void *frame,
			      void (*fn)(void), void *rvalue,
			      int flags, void *closure);

   Therefore on entry we have:

   x0 stack
   x1 frame
   x2 fn
   x3 rvalue
   x4 flags
   x5 closure
*/

	cfi_startproc
CNAME(ffi_call_SYSV):
	/* Use a stack frame allocated by our caller.  */
	cfi_def_cfa(x1, 32);
	stp	x29, x30, [x1]
	mov	x29, x1
	mov	sp, x0
	cfi_def_cfa_register(x29)
	cfi_rel_offset (x29, 0)
	cfi_rel_offset (x30, 8)

	mov	x9, x2			/* save fn */
	mov	x8, x3			/* install structure return */
#ifdef FFI_GO_CLOSURES
	mov	x18, x5			/* install static chain */
#endif
	stp	x3, x4, [x29, #16]	/* save rvalue and flags */

	/* Load the vector argument passing registers, if necessary.  */
	tbz	w4, #AARCH64_FLAG_ARG_V_BIT, 1f
	ldp     q0, q1, [sp, #0]
	ldp     q2, q3, [sp, #32]
	ldp     q4, q5, [sp, #64]
	ldp     q6, q7, [sp, #96]
1:
	/* Load the core argument passing registers, including
	   the structure return pointer.  */
	ldp     x0, x1, [sp, #16*N_V_ARG_REG + 0]
	ldp     x2, x3, [sp, #16*N_V_ARG_REG + 16]
	ldp     x4, x5, [sp, #16*N_V_ARG_REG + 32]
	ldp     x6, x7, [sp, #16*N_V_ARG_REG + 48]

	/* Deallocate the context, leaving the stacked arguments.  */
	add	sp, sp, #CALL_CONTEXT_SIZE

	blr     x9			/* call fn */

	ldp	x3, x4, [x29, #16]	/* reload rvalue and flags */

	/* Partially deconstruct the stack frame.  */
	mov     sp, x29
	cfi_def_cfa_register (sp)
	ldp     x29, x30, [x29]

	/* Save the return value as directed.  */
	adr	x5, 0f
	and	w4, w4, #AARCH64_RET_MASK
	add	x5, x5, x4, lsl #3
	br	x5

	/* Note that each table entry is 2 insns, and thus 8 bytes.
	   For integer data, note that we're storing into ffi_arg
	   and therefore we want to extend to 64 bits; these types
	   have two consecutive entries allocated for them.  */
	.align	4
0:	ret				/* VOID */
	nop
1:	str	x0, [x3]		/* INT64 */
	ret
2:	stp	x0, x1, [x3]		/* INT128 */
	ret
3:	brk	#1000			/* UNUSED */
	ret
4:	brk	#1000			/* UNUSED */
	ret
5:	brk	#1000			/* UNUSED */
	ret
6:	brk	#1000			/* UNUSED */
	ret
7:	brk	#1000			/* UNUSED */
	ret
8:	st4	{ v0.s, v1.s, v2.s, v3.s }[0], [x3]	/* S4 */
	ret
9:	st3	{ v0.s, v1.s, v2.s }[0], [x3]	/* S3 */
	ret
10:	stp	s0, s1, [x3]		/* S2 */
	ret
11:	str	s0, [x3]		/* S1 */
	ret
12:	st4	{ v0.d, v1.d, v2.d, v3.d }[0], [x3]	/* D4 */
	ret
13:	st3	{ v0.d, v1.d, v2.d }[0], [x3]	/* D3 */
	ret
14:	stp	d0, d1, [x3]		/* D2 */
	ret
15:	str	d0, [x3]		/* D1 */
	ret
16:	str	q3, [x3, #48]		/* Q4 */
	nop
17:	str	q2, [x3, #32]		/* Q3 */
	nop
18:	stp	q0, q1, [x3]		/* Q2 */
	ret
19:	str	q0, [x3]		/* Q1 */
	ret
20:	uxtb	w0, w0			/* UINT8 */
	str	x0, [x3]
21:	ret				/* reserved */
	nop
22:	uxth	w0, w0			/* UINT16 */
	str	x0, [x3]
23:	ret				/* reserved */
	nop
24:	mov	w0, w0			/* UINT32 */
	str	x0, [x3]
25:	ret				/* reserved */
	nop
26:	sxtb	x0, w0			/* SINT8 */
	str	x0, [x3]
27:	ret				/* reserved */
	nop
28:	sxth	x0, w0			/* SINT16 */
	str	x0, [x3]
29:	ret				/* reserved */
	nop
30:	sxtw	x0, w0			/* SINT32 */
	str	x0, [x3]
31:	ret				/* reserved */
	nop

	cfi_endproc

	.globl	CNAME(ffi_call_SYSV)
#ifdef __ELF__
	.type	CNAME(ffi_call_SYSV), #function
	.hidden	CNAME(ffi_call_SYSV)
	.size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV)
#endif

/* ffi_closure_SYSV

   Closure invocation glue. This is the low level code invoked directly by
   the closure trampoline to setup and call a closure.

   On entry x17 points to a struct ffi_closure, x16 has been clobbered
   all other registers are preserved.

   We allocate a call context and save the argument passing registers,
   then invoked the generic C ffi_closure_SYSV_inner() function to do all
   the real work, on return we load the result passing registers back from
   the call context.
*/

#define ffi_closure_SYSV_FS (8*2 + CALL_CONTEXT_SIZE + 64)

	.align 4
CNAME(ffi_closure_SYSV_V):
	cfi_startproc
	stp     x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	cfi_rel_offset (x29, 0)
	cfi_rel_offset (x30, 8)

	/* Save the argument passing vector registers.  */
	stp     q0, q1, [sp, #16 + 0]
	stp     q2, q3, [sp, #16 + 32]
	stp     q4, q5, [sp, #16 + 64]
	stp     q6, q7, [sp, #16 + 96]
	b	0f
	cfi_endproc

	.globl	CNAME(ffi_closure_SYSV_V)
#ifdef __ELF__
	.type	CNAME(ffi_closure_SYSV_V), #function
	.hidden	CNAME(ffi_closure_SYSV_V)
	.size	CNAME(ffi_closure_SYSV_V), . - CNAME(ffi_closure_SYSV_V)
#endif

	.align	4
	cfi_startproc
CNAME(ffi_closure_SYSV):
	stp     x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	cfi_rel_offset (x29, 0)
	cfi_rel_offset (x30, 8)
0:
	mov     x29, sp

	/* Save the argument passing core registers.  */
	stp     x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0]
	stp     x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16]
	stp     x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32]
	stp     x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48]

	/* Load ffi_closure_inner arguments.  */
	ldp	x0, x1, [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET]	/* load cif, fn */
	ldr	x2, [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET+16]	/* load user_data */
.Ldo_closure:
	add	x3, sp, #16				/* load context */
	add	x4, sp, #ffi_closure_SYSV_FS		/* load stack */
	add	x5, sp, #16+CALL_CONTEXT_SIZE		/* load rvalue */
	mov	x6, x8					/* load struct_rval */
	bl      CNAME(ffi_closure_SYSV_inner)

	/* Load the return value as directed.  */
	adr	x1, 0f
	and	w0, w0, #AARCH64_RET_MASK
	add	x1, x1, x0, lsl #3
	add	x3, sp, #16+CALL_CONTEXT_SIZE
	br	x1

	/* Note that each table entry is 2 insns, and thus 8 bytes.  */
	.align	4
0:	b	99f			/* VOID */
	nop
1:	ldr	x0, [x3]		/* INT64 */
	b	99f
2:	ldp	x0, x1, [x3]		/* INT128 */
	b	99f
3:	brk	#1000			/* UNUSED */
	nop
4:	brk	#1000			/* UNUSED */
	nop
5:	brk	#1000			/* UNUSED */
	nop
6:	brk	#1000			/* UNUSED */
	nop
7:	brk	#1000			/* UNUSED */
	nop
8:	ldr	s3, [x3, #12]		/* S4 */
	nop
9:	ldr	s2, [x2, #8]		/* S3 */
	nop
10:	ldp	s0, s1, [x3]		/* S2 */
	b	99f
11:	ldr	s0, [x3]		/* S1 */
	b	99f
12:	ldr	d3, [x3, #24]		/* D4 */
	nop
13:	ldr	d2, [x3, #16]		/* D3 */
	nop
14:	ldp	d0, d1, [x3]		/* D2 */
	b	99f
15:	ldr	d0, [x3]		/* D1 */
	b	99f
16:	ldr	q3, [x3, #48]		/* Q4 */
	nop
17:	ldr	q2, [x3, #32]		/* Q3 */
	nop
18:	ldp	q0, q1, [x3]		/* Q2 */
	b	99f
19:	ldr	q0, [x3]		/* Q1 */
	b	99f
20:	ldrb	w0, [x3, #BE(7)]	/* UINT8 */
	b	99f
21:	brk	#1000			/* reserved */
	nop
22:	ldrh	w0, [x3, #BE(6)]	/* UINT16 */
	b	99f
23:	brk	#1000			/* reserved */
	nop
24:	ldr	w0, [x3, #BE(4)]	/* UINT32 */
	b	99f
25:	brk	#1000			/* reserved */
	nop
26:	ldrsb	x0, [x3, #BE(7)]	/* SINT8 */
	b	99f
27:	brk	#1000			/* reserved */
	nop
28:	ldrsh	x0, [x3, #BE(6)]	/* SINT16 */
	b	99f
29:	brk	#1000			/* reserved */
	nop
30:	ldrsw	x0, [x3, #BE(4)]	/* SINT32 */
	nop
31:					/* reserved */
99:	ldp     x29, x30, [sp], #ffi_closure_SYSV_FS
	cfi_adjust_cfa_offset (-ffi_closure_SYSV_FS)
	cfi_restore (x29)
	cfi_restore (x30)
	ret
	cfi_endproc

	.globl	CNAME(ffi_closure_SYSV)
#ifdef __ELF__
	.type	CNAME(ffi_closure_SYSV), #function
	.hidden	CNAME(ffi_closure_SYSV)
	.size	CNAME(ffi_closure_SYSV), . - CNAME(ffi_closure_SYSV)
#endif

#if FFI_EXEC_TRAMPOLINE_TABLE
    .align 12
CNAME(ffi_closure_trampoline_table_page):
    .rept 16384 / FFI_TRAMPOLINE_SIZE
    adr	x17, -16384
    adr	x16, -16380
    ldr x16, [x16]
    ldr x17, [x17]
    br	x16
    .endr
    
    .globl CNAME(ffi_closure_trampoline_table_page)
    #ifdef __ELF__
    	.type	CNAME(ffi_closure_trampoline_table_page), #function
    	.hidden	CNAME(ffi_closure_trampoline_table_page)
    	.size	CNAME(ffi_closure_trampoline_table_page), . - CNAME(ffi_closure_trampoline_table_page)
    #endif
#endif

#ifdef FFI_GO_CLOSURES
	.align 4
CNAME(ffi_go_closure_SYSV_V):
	cfi_startproc
	stp     x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	cfi_rel_offset (x29, 0)
	cfi_rel_offset (x30, 8)

	/* Save the argument passing vector registers.  */
	stp     q0, q1, [sp, #16 + 0]
	stp     q2, q3, [sp, #16 + 32]
	stp     q4, q5, [sp, #16 + 64]
	stp     q6, q7, [sp, #16 + 96]
	b	0f
	cfi_endproc

	.globl	CNAME(ffi_go_closure_SYSV_V)
#ifdef __ELF__
	.type	CNAME(ffi_go_closure_SYSV_V), #function
	.hidden	CNAME(ffi_go_closure_SYSV_V)
	.size	CNAME(ffi_go_closure_SYSV_V), . - CNAME(ffi_go_closure_SYSV_V)
#endif

	.align	4
	cfi_startproc
CNAME(ffi_go_closure_SYSV):
	stp     x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	cfi_rel_offset (x29, 0)
	cfi_rel_offset (x30, 8)
0:
	mov     x29, sp

	/* Save the argument passing core registers.  */
	stp     x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0]
	stp     x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16]
	stp     x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32]
	stp     x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48]

	/* Load ffi_closure_inner arguments.  */
	ldp	x0, x1, [x18, #8]			/* load cif, fn */
	mov	x2, x18					/* load user_data */
	b	.Ldo_closure
	cfi_endproc

	.globl	CNAME(ffi_go_closure_SYSV)
#ifdef __ELF__
	.type	CNAME(ffi_go_closure_SYSV), #function
	.hidden	CNAME(ffi_go_closure_SYSV)
	.size	CNAME(ffi_go_closure_SYSV), . - CNAME(ffi_go_closure_SYSV)
#endif
#endif /* FFI_GO_CLOSURES */

#if defined __ELF__ && defined __linux__
	.section .note.GNU-stack,"",%progbits
#endif
Commit	Line	Data
b4b575ce AG	1	/* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
	2
	3	Permission is hereby granted, free of charge, to any person obtaining
	4	a copy of this software and associated documentation files (the
	5	``Software''), to deal in the Software without restriction, including
	6	without limitation the rights to use, copy, modify, merge, publish,
	7	distribute, sublicense, and/or sell copies of the Software, and to
	8	permit persons to whom the Software is furnished to do so, subject to
	9	the following conditions:
	10
	11	The above copyright notice and this permission notice shall be
	12	included in all copies or substantial portions of the Software.
	13
	14	THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
	15	EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	16	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	17	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
	18	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
	19	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
	20	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
	21
	22	#define LIBFFI_ASM
	23	#include <fficonfig.h>
	24	#include <ffi.h>
b1760f7f RH	25	#include <ffi_cfi.h>
	26	#include "internal.h"
	27
	28	#ifdef HAVE_MACHINE_ASM_H
	29	#include <machine/asm.h>
	30	#else
	31	#ifdef __USER_LABEL_PREFIX__
	32	#define CONCAT1(a, b) CONCAT2(a, b)
	33	#define CONCAT2(a, b) a ## b
	34
	35	/* Use the right prefix for global labels. */
	36	#define CNAME(x) CONCAT1 (__USER_LABEL_PREFIX__, x)
	37	#else
	38	#define CNAME(x) x
	39	#endif
	40	#endif
	41
	42	#ifdef __AARCH64EB__
	43	# define BE(X) X
	44	#else
	45	# define BE(X) 0
	46	#endif
	47
	48	.text
	49	.align 4
	50
	51	/* ffi_call_SYSV
	52	extern void ffi_call_SYSV (void stack, void frame,
	53	void (fn)(void), void rvalue,
	54	int flags, void *closure);
b4b575ce AG	55
	56	Therefore on entry we have:
	57
b1760f7f RH	58	x0 stack
	59	x1 frame
	60	x2 fn
	61	x3 rvalue
	62	x4 flags
	63	x5 closure
	64	*/
	65
	66	cfi_startproc
	67	CNAME(ffi_call_SYSV):
	68	/* Use a stack frame allocated by our caller. */
	69	cfi_def_cfa(x1, 32);
	70	stp x29, x30, [x1]
	71	mov x29, x1
	72	mov sp, x0
	73	cfi_def_cfa_register(x29)
	74	cfi_rel_offset (x29, 0)
	75	cfi_rel_offset (x30, 8)
	76
	77	mov x9, x2 /* save fn */
	78	mov x8, x3 /* install structure return */
	79	#ifdef FFI_GO_CLOSURES
	80	mov x18, x5 /* install static chain */
	81	#endif
	82	stp x3, x4, [x29, #16] /* save rvalue and flags */
	83
	84	/* Load the vector argument passing registers, if necessary. */
	85	tbz w4, #AARCH64_FLAG_ARG_V_BIT, 1f
	86	ldp q0, q1, [sp, #0]
	87	ldp q2, q3, [sp, #32]
	88	ldp q4, q5, [sp, #64]
	89	ldp q6, q7, [sp, #96]
b4b575ce	90	1:
b1760f7f RH	91	/* Load the core argument passing registers, including
	92	the structure return pointer. */
	93	ldp x0, x1, [sp, #16*N_V_ARG_REG + 0]
	94	ldp x2, x3, [sp, #16*N_V_ARG_REG + 16]
	95	ldp x4, x5, [sp, #16*N_V_ARG_REG + 32]
	96	ldp x6, x7, [sp, #16*N_V_ARG_REG + 48]
b4b575ce	97
b1760f7f RH	98	/* Deallocate the context, leaving the stacked arguments. */
b1760f7f RH	99	add sp, sp, #CALL_CONTEXT_SIZE
b4b575ce	100
b1760f7f	101	blr x9 /* call fn */
b4b575ce	102
b1760f7f	103	ldp x3, x4, [x29, #16] /* reload rvalue and flags */
b4b575ce	104
b1760f7f RH	105	/* Partially deconstruct the stack frame. */
	106	mov sp, x29
	107	cfi_def_cfa_register (sp)
	108	ldp x29, x30, [x29]
	109
	110	/* Save the return value as directed. */
	111	adr x5, 0f
	112	and w4, w4, #AARCH64_RET_MASK
	113	add x5, x5, x4, lsl #3
	114	br x5
	115
	116	/* Note that each table entry is 2 insns, and thus 8 bytes.
	117	For integer data, note that we're storing into ffi_arg
	118	and therefore we want to extend to 64 bits; these types
	119	have two consecutive entries allocated for them. */
	120	.align 4
	121	0: ret /* VOID */
	122	nop
	123	1: str x0, [x3] /* INT64 */
	124	ret
	125	2: stp x0, x1, [x3] /* INT128 */
	126	ret
	127	3: brk #1000 /* UNUSED */
	128	ret
	129	4: brk #1000 /* UNUSED */
	130	ret
	131	5: brk #1000 /* UNUSED */
	132	ret
	133	6: brk #1000 /* UNUSED */
	134	ret
	135	7: brk #1000 /* UNUSED */
	136	ret
ed4bf691	137	8: st4 { v0.s, v1.s, v2.s, v3.s }[0], [x3] /* S4 */
b1760f7f	138	ret
ed4bf691	139	9: st3 { v0.s, v1.s, v2.s }[0], [x3] /* S3 */
b1760f7f RH	140	ret
	141	10: stp s0, s1, [x3] /* S2 */
	142	ret
	143	11: str s0, [x3] /* S1 */
	144	ret
ed4bf691	145	12: st4 { v0.d, v1.d, v2.d, v3.d }[0], [x3] /* D4 */
b1760f7f	146	ret
ed4bf691	147	13: st3 { v0.d, v1.d, v2.d }[0], [x3] /* D3 */
b1760f7f RH	148	ret
	149	14: stp d0, d1, [x3] /* D2 */
	150	ret
	151	15: str d0, [x3] /* D1 */
	152	ret
	153	16: str q3, [x3, #48] /* Q4 */
	154	nop
	155	17: str q2, [x3, #32] /* Q3 */
	156	nop
	157	18: stp q0, q1, [x3] /* Q2 */
	158	ret
	159	19: str q0, [x3] /* Q1 */
	160	ret
	161	20: uxtb w0, w0 /* UINT8 */
	162	str x0, [x3]
	163	21: ret /* reserved */
	164	nop
	165	22: uxth w0, w0 /* UINT16 */
	166	str x0, [x3]
	167	23: ret /* reserved */
	168	nop
	169	24: mov w0, w0 /* UINT32 */
	170	str x0, [x3]
	171	25: ret /* reserved */
	172	nop
	173	26: sxtb x0, w0 /* SINT8 */
	174	str x0, [x3]
	175	27: ret /* reserved */
	176	nop
	177	28: sxth x0, w0 /* SINT16 */
	178	str x0, [x3]
	179	29: ret /* reserved */
	180	nop
	181	30: sxtw x0, w0 /* SINT32 */
	182	str x0, [x3]
	183	31: ret /* reserved */
	184	nop
	185
	186	cfi_endproc
	187
	188	.globl CNAME(ffi_call_SYSV)
	189	#ifdef __ELF__
	190	.type CNAME(ffi_call_SYSV), #function
	191	.hidden CNAME(ffi_call_SYSV)
	192	.size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV)
	193	#endif
b4b575ce AG	194
	195	/* ffi_closure_SYSV
	196
	197	Closure invocation glue. This is the low level code invoked directly by
	198	the closure trampoline to setup and call a closure.
	199
b1760f7f	200	On entry x17 points to a struct ffi_closure, x16 has been clobbered
b4b575ce AG	201	all other registers are preserved.
	202
	203	We allocate a call context and save the argument passing registers,
	204	then invoked the generic C ffi_closure_SYSV_inner() function to do all
	205	the real work, on return we load the result passing registers back from
	206	the call context.
b1760f7f RH	207	*/
	208
	209	#define ffi_closure_SYSV_FS (8*2 + CALL_CONTEXT_SIZE + 64)
	210
	211	.align 4
	212	CNAME(ffi_closure_SYSV_V):
	213	cfi_startproc
	214	stp x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	215	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	216	cfi_rel_offset (x29, 0)
	217	cfi_rel_offset (x30, 8)
	218
	219	/* Save the argument passing vector registers. */
	220	stp q0, q1, [sp, #16 + 0]
	221	stp q2, q3, [sp, #16 + 32]
	222	stp q4, q5, [sp, #16 + 64]
	223	stp q6, q7, [sp, #16 + 96]
	224	b 0f
	225	cfi_endproc
	226
	227	.globl CNAME(ffi_closure_SYSV_V)
	228	#ifdef __ELF__
	229	.type CNAME(ffi_closure_SYSV_V), #function
	230	.hidden CNAME(ffi_closure_SYSV_V)
	231	.size CNAME(ffi_closure_SYSV_V), . - CNAME(ffi_closure_SYSV_V)
	232	#endif
	233
	234	.align 4
	235	cfi_startproc
	236	CNAME(ffi_closure_SYSV):
	237	stp x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	238	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	239	cfi_rel_offset (x29, 0)
	240	cfi_rel_offset (x30, 8)
	241	0:
	242	mov x29, sp
	243
	244	/* Save the argument passing core registers. */
	245	stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0]
	246	stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16]
	247	stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32]
	248	stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48]
	249
	250	/* Load ffi_closure_inner arguments. */
ed4bf691 RH	251	ldp x0, x1, [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET] /* load cif, fn */
ed4bf691 RH	252	ldr x2, [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET+16] /* load user_data */
b1760f7f RH	253	.Ldo_closure:
	254	add x3, sp, #16 /* load context */
	255	add x4, sp, #ffi_closure_SYSV_FS /* load stack */
	256	add x5, sp, #16+CALL_CONTEXT_SIZE /* load rvalue */
	257	mov x6, x8 /* load struct_rval */
	258	bl CNAME(ffi_closure_SYSV_inner)
	259
	260	/* Load the return value as directed. */
	261	adr x1, 0f
	262	and w0, w0, #AARCH64_RET_MASK
	263	add x1, x1, x0, lsl #3
	264	add x3, sp, #16+CALL_CONTEXT_SIZE
	265	br x1
	266
	267	/* Note that each table entry is 2 insns, and thus 8 bytes. */
	268	.align 4
	269	0: b 99f /* VOID */
	270	nop
	271	1: ldr x0, [x3] /* INT64 */
	272	b 99f
	273	2: ldp x0, x1, [x3] /* INT128 */
	274	b 99f
	275	3: brk #1000 /* UNUSED */
	276	nop
	277	4: brk #1000 /* UNUSED */
	278	nop
	279	5: brk #1000 /* UNUSED */
	280	nop
	281	6: brk #1000 /* UNUSED */
	282	nop
	283	7: brk #1000 /* UNUSED */
	284	nop
	285	8: ldr s3, [x3, #12] /* S4 */
	286	nop
	287	9: ldr s2, [x2, #8] /* S3 */
	288	nop
	289	10: ldp s0, s1, [x3] /* S2 */
	290	b 99f
	291	11: ldr s0, [x3] /* S1 */
	292	b 99f
	293	12: ldr d3, [x3, #24] /* D4 */
	294	nop
	295	13: ldr d2, [x3, #16] /* D3 */
	296	nop
	297	14: ldp d0, d1, [x3] /* D2 */
	298	b 99f
	299	15: ldr d0, [x3] /* D1 */
	300	b 99f
	301	16: ldr q3, [x3, #48] /* Q4 */
	302	nop
	303	17: ldr q2, [x3, #32] /* Q3 */
	304	nop
	305	18: ldp q0, q1, [x3] /* Q2 */
	306	b 99f
	307	19: ldr q0, [x3] /* Q1 */
	308	b 99f
	309	20: ldrb w0, [x3, #BE(7)] /* UINT8 */
	310	b 99f
	311	21: brk #1000 /* reserved */
	312	nop
	313	22: ldrh w0, [x3, #BE(6)] /* UINT16 */
	314	b 99f
	315	23: brk #1000 /* reserved */
	316	nop
317	24: ldr w0, [x3, #BE(4)] /* UINT32 */
318	b 99f
319	25: brk #1000 /* reserved */
320	nop
321	26: ldrsb x0, [x3, #BE(7)] /* SINT8 */
322	b 99f
323	27: brk #1000 /* reserved */
324	nop
325	28: ldrsh x0, [x3, #BE(6)] /* SINT16 */
326	b 99f
327	29: brk #1000 /* reserved */
328	nop
329	30: ldrsw x0, [x3, #BE(4)] /* SINT32 */
330	nop
331	31: /* reserved */
332	99: ldp x29, x30, [sp], #ffi_closure_SYSV_FS
333	cfi_adjust_cfa_offset (-ffi_closure_SYSV_FS)
334	cfi_restore (x29)
335	cfi_restore (x30)
336	ret
337	cfi_endproc
338
339	.globl CNAME(ffi_closure_SYSV)
340	#ifdef __ELF__
341	.type CNAME(ffi_closure_SYSV), #function
342	.hidden CNAME(ffi_closure_SYSV)
343	.size CNAME(ffi_closure_SYSV), . - CNAME(ffi_closure_SYSV)
344	#endif
345
ed4bf691 RH	346	#if FFI_EXEC_TRAMPOLINE_TABLE
	347	.align 12
	348	CNAME(ffi_closure_trampoline_table_page):
	349	.rept 16384 / FFI_TRAMPOLINE_SIZE
	350	adr x17, -16384
	351	adr x16, -16380
	352	ldr x16, [x16]
	353	ldr x17, [x17]
	354	br x16
	355	.endr
	356
	357	.globl CNAME(ffi_closure_trampoline_table_page)
	358	#ifdef __ELF__
	359	.type CNAME(ffi_closure_trampoline_table_page), #function
	360	.hidden CNAME(ffi_closure_trampoline_table_page)
	361	.size CNAME(ffi_closure_trampoline_table_page), . - CNAME(ffi_closure_trampoline_table_page)
	362	#endif
	363	#endif
	364
b1760f7f RH	365	#ifdef FFI_GO_CLOSURES
	366	.align 4
	367	CNAME(ffi_go_closure_SYSV_V):
	368	cfi_startproc
	369	stp x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	370	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	371	cfi_rel_offset (x29, 0)
	372	cfi_rel_offset (x30, 8)
	373
	374	/* Save the argument passing vector registers. */
	375	stp q0, q1, [sp, #16 + 0]
	376	stp q2, q3, [sp, #16 + 32]
	377	stp q4, q5, [sp, #16 + 64]
	378	stp q6, q7, [sp, #16 + 96]
	379	b 0f
	380	cfi_endproc
	381
	382	.globl CNAME(ffi_go_closure_SYSV_V)
	383	#ifdef __ELF__
	384	.type CNAME(ffi_go_closure_SYSV_V), #function
	385	.hidden CNAME(ffi_go_closure_SYSV_V)
	386	.size CNAME(ffi_go_closure_SYSV_V), . - CNAME(ffi_go_closure_SYSV_V)
	387	#endif
	388
	389	.align 4
	390	cfi_startproc
	391	CNAME(ffi_go_closure_SYSV):
	392	stp x29, x30, [sp, #-ffi_closure_SYSV_FS]!
	393	cfi_adjust_cfa_offset (ffi_closure_SYSV_FS)
	394	cfi_rel_offset (x29, 0)
	395	cfi_rel_offset (x30, 8)
	396	0:
	397	mov x29, sp
	398
	399	/* Save the argument passing core registers. */
	400	stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0]
	401	stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16]
	402	stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32]
	403	stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48]
	404
	405	/* Load ffi_closure_inner arguments. */
	406	ldp x0, x1, [x18, #8] /* load cif, fn */
	407	mov x2, x18 /* load user_data */
	408	b .Ldo_closure
	409	cfi_endproc
	410
	411	.globl CNAME(ffi_go_closure_SYSV)
	412	#ifdef __ELF__
	413	.type CNAME(ffi_go_closure_SYSV), #function
	414	.hidden CNAME(ffi_go_closure_SYSV)
	415	.size CNAME(ffi_go_closure_SYSV), . - CNAME(ffi_go_closure_SYSV)
	416	#endif
	417	#endif /* FFI_GO_CLOSURES */
	418
	419	#if defined __ELF__ && defined __linux__
	420	.section .note.GNU-stack,"",%progbits
	421	#endif
b4b575ce	422