1 /* -----------------------------------------------------------------------
2 ffi.c - Copyright (C) 2011 Anthony Green
3 Copyright (C) 2011 Kyle Moffett
4 Copyright (C) 2008 Red Hat, Inc
5 Copyright (C) 2007, 2008 Free Software Foundation, Inc
6 Copyright (c) 1998 Geoffrey Keating
8 PowerPC Foreign Function Interface
10 Permission is hereby granted, free of charge, to any person obtaining
11 a copy of this software and associated documentation files (the
12 ``Software''), to deal in the Software without restriction, including
13 without limitation the rights to use, copy, modify, merge, publish,
14 distribute, sublicense, and/or sell copies of the Software, and to
15 permit persons to whom the Software is furnished to do so, subject to
16 the following conditions:
18 The above copyright notice and this permission notice shall be included
19 in all copies or substantial portions of the Software.
21 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
24 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
25 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
26 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27 OTHER DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
31 #include <ffi_common.h>
37 extern void ffi_closure_SYSV (void);
38 extern void FFI_HIDDEN
ffi_closure_LINUX64 (void);
41 /* The assembly depends on these exact flags. */
42 FLAG_RETURNS_SMST
= 1 << (31-31), /* Used for FFI_SYSV small structs. */
43 FLAG_RETURNS_NOTHING
= 1 << (31-30), /* These go in cr7 */
45 FLAG_RETURNS_FP
= 1 << (31-29),
47 FLAG_RETURNS_64BITS
= 1 << (31-28),
49 FLAG_RETURNS_128BITS
= 1 << (31-27), /* cr6 */
51 FLAG_ARG_NEEDS_COPY
= 1 << (31- 7),
53 FLAG_FP_ARGUMENTS
= 1 << (31- 6), /* cr1.eq; specified by ABI */
55 FLAG_4_GPR_ARGUMENTS
= 1 << (31- 5),
56 FLAG_RETVAL_REFERENCE
= 1 << (31- 4)
59 /* About the SYSV ABI. */
60 #define ASM_NEEDS_REGISTERS 4
61 #define NUM_GPR_ARG_REGISTERS 8
63 # define NUM_FPR_ARG_REGISTERS 8
66 /* ffi_prep_args_SYSV is called by the assembly routine once stack space
67 has been allocated for the function's arguments.
69 The stack layout we want looks like this:
71 | Return address from ffi_call_SYSV 4bytes | higher addresses
72 |--------------------------------------------|
73 | Previous backchain pointer 4 | stack pointer here
74 |--------------------------------------------|<+ <<< on entry to
75 | Saved r28-r31 4*4 | | ffi_call_SYSV
76 |--------------------------------------------| |
77 | GPR registers r3-r10 8*4 | | ffi_call_SYSV
78 |--------------------------------------------| |
79 | FPR registers f1-f8 (optional) 8*8 | |
80 |--------------------------------------------| | stack |
81 | Space for copied structures | | grows |
82 |--------------------------------------------| | down V
83 | Parameters that didn't fit in registers | |
84 |--------------------------------------------| | lower addresses
85 | Space for callee's LR 4 | |
86 |--------------------------------------------| | stack pointer here
87 | Current backchain pointer 4 |-/ during
88 |--------------------------------------------| <<< ffi_call_SYSV
93 ffi_prep_args_SYSV (extended_cif
*ecif
, unsigned *const stack
)
95 const unsigned bytes
= ecif
->cif
->bytes
;
96 const unsigned flags
= ecif
->cif
->flags
;
106 /* 'stacktop' points at the previous backchain pointer. */
109 /* 'gpr_base' points at the space for gpr3, and grows upwards as
110 we use GPR registers. */
115 /* 'fpr_base' points at the space for fpr1, and grows upwards as
116 we use FPR registers. */
121 /* 'copy_space' grows down as we put structures in it. It should
122 stay 16-byte aligned. */
125 /* 'next_arg' grows up as we put parameters in it. */
142 size_t struct_copy_size
;
145 stacktop
.c
= (char *) stack
+ bytes
;
146 gpr_base
.u
= stacktop
.u
- ASM_NEEDS_REGISTERS
- NUM_GPR_ARG_REGISTERS
;
150 fpr_base
.d
= gpr_base
.d
- NUM_FPR_ARG_REGISTERS
;
152 copy_space
.c
= ((flags
& FLAG_FP_ARGUMENTS
) ? fpr_base
.c
: gpr_base
.c
);
154 copy_space
.c
= gpr_base
.c
;
156 next_arg
.u
= stack
+ 2;
158 /* Check that everything starts aligned properly. */
159 FFI_ASSERT (((unsigned long) (char *) stack
& 0xF) == 0);
160 FFI_ASSERT (((unsigned long) copy_space
.c
& 0xF) == 0);
161 FFI_ASSERT (((unsigned long) stacktop
.c
& 0xF) == 0);
162 FFI_ASSERT ((bytes
& 0xF) == 0);
163 FFI_ASSERT (copy_space
.c
>= next_arg
.c
);
165 /* Deal with return values that are actually pass-by-reference. */
166 if (flags
& FLAG_RETVAL_REFERENCE
)
168 *gpr_base
.u
++ = (unsigned long) (char *) ecif
->rvalue
;
172 /* Now for the arguments. */
173 p_argv
.v
= ecif
->avalue
;
174 for (ptr
= ecif
->cif
->arg_types
, i
= ecif
->cif
->nargs
;
176 i
--, ptr
++, p_argv
.v
++)
178 unsigned short typenum
= (*ptr
)->type
;
180 /* We may need to handle some values depending on ABI */
181 if (ecif
->cif
->abi
== FFI_LINUX_SOFT_FLOAT
) {
182 if (typenum
== FFI_TYPE_FLOAT
)
183 typenum
= FFI_TYPE_UINT32
;
184 if (typenum
== FFI_TYPE_DOUBLE
)
185 typenum
= FFI_TYPE_UINT64
;
186 if (typenum
== FFI_TYPE_LONGDOUBLE
)
187 typenum
= FFI_TYPE_UINT128
;
188 } else if (ecif
->cif
->abi
!= FFI_LINUX
) {
189 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
190 if (typenum
== FFI_TYPE_LONGDOUBLE
)
191 typenum
= FFI_TYPE_STRUCT
;
195 /* Now test the translated value */
199 /* With FFI_LINUX_SOFT_FLOAT floats are handled like UINT32. */
200 double_tmp
= **p_argv
.f
;
201 if (fparg_count
>= NUM_FPR_ARG_REGISTERS
)
203 *next_arg
.f
= (float) double_tmp
;
208 *fpr_base
.d
++ = double_tmp
;
210 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
213 case FFI_TYPE_DOUBLE
:
214 /* With FFI_LINUX_SOFT_FLOAT doubles are handled like UINT64. */
215 double_tmp
= **p_argv
.d
;
217 if (fparg_count
>= NUM_FPR_ARG_REGISTERS
)
219 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
220 && intarg_count
% 2 != 0)
225 *next_arg
.d
= double_tmp
;
229 *fpr_base
.d
++ = double_tmp
;
231 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
234 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
235 case FFI_TYPE_LONGDOUBLE
:
236 double_tmp
= (*p_argv
.d
)[0];
238 if (fparg_count
>= NUM_FPR_ARG_REGISTERS
- 1)
240 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
241 && intarg_count
% 2 != 0)
246 *next_arg
.d
= double_tmp
;
248 double_tmp
= (*p_argv
.d
)[1];
249 *next_arg
.d
= double_tmp
;
254 *fpr_base
.d
++ = double_tmp
;
255 double_tmp
= (*p_argv
.d
)[1];
256 *fpr_base
.d
++ = double_tmp
;
260 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
263 #endif /* have FPRs */
266 * The soft float ABI for long doubles works like this, a long double
267 * is passed in four consecutive GPRs if available. A maximum of 2
268 * long doubles can be passed in gprs. If we do not have 4 GPRs
269 * left, the long double is passed on the stack, 4-byte aligned.
271 case FFI_TYPE_UINT128
: {
272 unsigned int int_tmp
= (*p_argv
.ui
)[0];
274 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
- 3) {
275 if (intarg_count
< NUM_GPR_ARG_REGISTERS
)
276 intarg_count
+= NUM_GPR_ARG_REGISTERS
- intarg_count
;
277 *(next_arg
.u
++) = int_tmp
;
278 for (ii
= 1; ii
< 4; ii
++) {
279 int_tmp
= (*p_argv
.ui
)[ii
];
280 *(next_arg
.u
++) = int_tmp
;
283 *(gpr_base
.u
++) = int_tmp
;
284 for (ii
= 1; ii
< 4; ii
++) {
285 int_tmp
= (*p_argv
.ui
)[ii
];
286 *(gpr_base
.u
++) = int_tmp
;
293 case FFI_TYPE_UINT64
:
294 case FFI_TYPE_SINT64
:
295 if (intarg_count
== NUM_GPR_ARG_REGISTERS
-1)
297 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
)
299 if (intarg_count
% 2 != 0)
304 *next_arg
.ll
= **p_argv
.ll
;
309 /* whoops: abi states only certain register pairs
310 * can be used for passing long long int
311 * specifically (r3,r4), (r5,r6), (r7,r8),
312 * (r9,r10) and if next arg is long long but
313 * not correct starting register of pair then skip
314 * until the proper starting register
316 if (intarg_count
% 2 != 0)
321 *gpr_base
.ll
++ = **p_argv
.ll
;
326 case FFI_TYPE_STRUCT
:
327 struct_copy_size
= ((*ptr
)->size
+ 15) & ~0xF;
328 copy_space
.c
-= struct_copy_size
;
329 memcpy (copy_space
.c
, *p_argv
.c
, (*ptr
)->size
);
331 gprvalue
= (unsigned long) copy_space
.c
;
333 FFI_ASSERT (copy_space
.c
> next_arg
.c
);
334 FFI_ASSERT (flags
& FLAG_ARG_NEEDS_COPY
);
338 gprvalue
= **p_argv
.uc
;
341 gprvalue
= **p_argv
.sc
;
343 case FFI_TYPE_UINT16
:
344 gprvalue
= **p_argv
.us
;
346 case FFI_TYPE_SINT16
:
347 gprvalue
= **p_argv
.ss
;
351 case FFI_TYPE_UINT32
:
352 case FFI_TYPE_SINT32
:
353 case FFI_TYPE_POINTER
:
355 gprvalue
= **p_argv
.ui
;
358 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
)
359 *next_arg
.u
++ = gprvalue
;
361 *gpr_base
.u
++ = gprvalue
;
367 /* Check that we didn't overrun the stack... */
368 FFI_ASSERT (copy_space
.c
>= next_arg
.c
);
369 FFI_ASSERT (gpr_base
.u
<= stacktop
.u
- ASM_NEEDS_REGISTERS
);
371 FFI_ASSERT (fpr_base
.u
372 <= stacktop
.u
- ASM_NEEDS_REGISTERS
- NUM_GPR_ARG_REGISTERS
);
374 FFI_ASSERT (flags
& FLAG_4_GPR_ARGUMENTS
|| intarg_count
<= 4);
377 /* About the LINUX64 ABI. */
379 NUM_GPR_ARG_REGISTERS64
= 8,
380 NUM_FPR_ARG_REGISTERS64
= 13
382 enum { ASM_NEEDS_REGISTERS64
= 4 };
384 /* ffi_prep_args64 is called by the assembly routine once stack space
385 has been allocated for the function's arguments.
387 The stack layout we want looks like this:
389 | Ret addr from ffi_call_LINUX64 8bytes | higher addresses
390 |--------------------------------------------|
391 | CR save area 8bytes |
392 |--------------------------------------------|
393 | Previous backchain pointer 8 | stack pointer here
394 |--------------------------------------------|<+ <<< on entry to
395 | Saved r28-r31 4*8 | | ffi_call_LINUX64
396 |--------------------------------------------| |
397 | GPR registers r3-r10 8*8 | |
398 |--------------------------------------------| |
399 | FPR registers f1-f13 (optional) 13*8 | |
400 |--------------------------------------------| |
401 | Parameter save area | |
402 |--------------------------------------------| |
403 | TOC save area 8 | |
404 |--------------------------------------------| | stack |
405 | Linker doubleword 8 | | grows |
406 |--------------------------------------------| | down V
407 | Compiler doubleword 8 | |
408 |--------------------------------------------| | lower addresses
409 | Space for callee's LR 8 | |
410 |--------------------------------------------| |
412 |--------------------------------------------| | stack pointer here
413 | Current backchain pointer 8 |-/ during
414 |--------------------------------------------| <<< ffi_call_LINUX64
419 ffi_prep_args64 (extended_cif
*ecif
, unsigned long *const stack
)
421 const unsigned long bytes
= ecif
->cif
->bytes
;
422 const unsigned long flags
= ecif
->cif
->flags
;
431 /* 'stacktop' points at the previous backchain pointer. */
434 /* 'next_arg' points at the space for gpr3, and grows upwards as
435 we use GPR registers, then continues at rest. */
441 /* 'fpr_base' points at the space for fpr3, and grows upwards as
442 we use FPR registers. */
462 unsigned long gprvalue
;
464 stacktop
.c
= (char *) stack
+ bytes
;
465 gpr_base
.ul
= stacktop
.ul
- ASM_NEEDS_REGISTERS64
- NUM_GPR_ARG_REGISTERS64
;
466 gpr_end
.ul
= gpr_base
.ul
+ NUM_GPR_ARG_REGISTERS64
;
467 rest
.ul
= stack
+ 6 + NUM_GPR_ARG_REGISTERS64
;
468 fpr_base
.d
= gpr_base
.d
- NUM_FPR_ARG_REGISTERS64
;
470 next_arg
.ul
= gpr_base
.ul
;
472 /* Check that everything starts aligned properly. */
473 FFI_ASSERT (((unsigned long) (char *) stack
& 0xF) == 0);
474 FFI_ASSERT (((unsigned long) stacktop
.c
& 0xF) == 0);
475 FFI_ASSERT ((bytes
& 0xF) == 0);
477 /* Deal with return values that are actually pass-by-reference. */
478 if (flags
& FLAG_RETVAL_REFERENCE
)
479 *next_arg
.ul
++ = (unsigned long) (char *) ecif
->rvalue
;
481 /* Now for the arguments. */
482 p_argv
.v
= ecif
->avalue
;
483 for (ptr
= ecif
->cif
->arg_types
, i
= ecif
->cif
->nargs
;
485 i
--, ptr
++, p_argv
.v
++)
487 switch ((*ptr
)->type
)
490 double_tmp
= **p_argv
.f
;
491 *next_arg
.f
= (float) double_tmp
;
492 if (++next_arg
.ul
== gpr_end
.ul
)
493 next_arg
.ul
= rest
.ul
;
494 if (fparg_count
< NUM_FPR_ARG_REGISTERS64
)
495 *fpr_base
.d
++ = double_tmp
;
497 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
500 case FFI_TYPE_DOUBLE
:
501 double_tmp
= **p_argv
.d
;
502 *next_arg
.d
= double_tmp
;
503 if (++next_arg
.ul
== gpr_end
.ul
)
504 next_arg
.ul
= rest
.ul
;
505 if (fparg_count
< NUM_FPR_ARG_REGISTERS64
)
506 *fpr_base
.d
++ = double_tmp
;
508 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
511 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
512 case FFI_TYPE_LONGDOUBLE
:
513 double_tmp
= (*p_argv
.d
)[0];
514 *next_arg
.d
= double_tmp
;
515 if (++next_arg
.ul
== gpr_end
.ul
)
516 next_arg
.ul
= rest
.ul
;
517 if (fparg_count
< NUM_FPR_ARG_REGISTERS64
)
518 *fpr_base
.d
++ = double_tmp
;
520 double_tmp
= (*p_argv
.d
)[1];
521 *next_arg
.d
= double_tmp
;
522 if (++next_arg
.ul
== gpr_end
.ul
)
523 next_arg
.ul
= rest
.ul
;
524 if (fparg_count
< NUM_FPR_ARG_REGISTERS64
)
525 *fpr_base
.d
++ = double_tmp
;
527 FFI_ASSERT (__LDBL_MANT_DIG__
== 106);
528 FFI_ASSERT (flags
& FLAG_FP_ARGUMENTS
);
532 case FFI_TYPE_STRUCT
:
533 words
= ((*ptr
)->size
+ 7) / 8;
534 if (next_arg
.ul
>= gpr_base
.ul
&& next_arg
.ul
+ words
> gpr_end
.ul
)
536 size_t first
= gpr_end
.c
- next_arg
.c
;
537 memcpy (next_arg
.c
, *p_argv
.c
, first
);
538 memcpy (rest
.c
, *p_argv
.c
+ first
, (*ptr
)->size
- first
);
539 next_arg
.c
= rest
.c
+ words
* 8 - first
;
543 char *where
= next_arg
.c
;
545 /* Structures with size less than eight bytes are passed
547 if ((*ptr
)->size
< 8)
548 where
+= 8 - (*ptr
)->size
;
550 memcpy (where
, *p_argv
.c
, (*ptr
)->size
);
551 next_arg
.ul
+= words
;
552 if (next_arg
.ul
== gpr_end
.ul
)
553 next_arg
.ul
= rest
.ul
;
558 gprvalue
= **p_argv
.uc
;
561 gprvalue
= **p_argv
.sc
;
563 case FFI_TYPE_UINT16
:
564 gprvalue
= **p_argv
.us
;
566 case FFI_TYPE_SINT16
:
567 gprvalue
= **p_argv
.ss
;
569 case FFI_TYPE_UINT32
:
570 gprvalue
= **p_argv
.ui
;
573 case FFI_TYPE_SINT32
:
574 gprvalue
= **p_argv
.si
;
577 case FFI_TYPE_UINT64
:
578 case FFI_TYPE_SINT64
:
579 case FFI_TYPE_POINTER
:
580 gprvalue
= **p_argv
.ul
;
582 *next_arg
.ul
++ = gprvalue
;
583 if (next_arg
.ul
== gpr_end
.ul
)
584 next_arg
.ul
= rest
.ul
;
589 FFI_ASSERT (flags
& FLAG_4_GPR_ARGUMENTS
590 || (next_arg
.ul
>= gpr_base
.ul
591 && next_arg
.ul
<= gpr_base
.ul
+ 4));
596 /* Perform machine dependent cif processing */
598 ffi_prep_cif_machdep (ffi_cif
*cif
)
600 /* All this is for the SYSV and LINUX64 ABI. */
604 int fparg_count
= 0, intarg_count
= 0;
606 unsigned struct_copy_size
= 0;
607 unsigned type
= cif
->rtype
->type
;
608 unsigned size
= cif
->rtype
->size
;
610 if (cif
->abi
!= FFI_LINUX64
)
612 /* All the machine-independent calculation of cif->bytes will be wrong.
613 Redo the calculation for SYSV. */
615 /* Space for the frame pointer, callee's LR, and the asm's temp regs. */
616 bytes
= (2 + ASM_NEEDS_REGISTERS
) * sizeof (int);
618 /* Space for the GPR registers. */
619 bytes
+= NUM_GPR_ARG_REGISTERS
* sizeof (int);
625 /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
627 bytes
= (6 + ASM_NEEDS_REGISTERS64
) * sizeof (long);
629 /* Space for the mandatory parm save area and general registers. */
630 bytes
+= 2 * NUM_GPR_ARG_REGISTERS64
* sizeof (long);
633 /* Return value handling. The rules for SYSV are as follows:
634 - 32-bit (or less) integer values are returned in gpr3;
635 - Structures of size <= 4 bytes also returned in gpr3;
636 - 64-bit integer values and structures between 5 and 8 bytes are returned
638 - Single/double FP values are returned in fpr1;
639 - Larger structures are allocated space and a pointer is passed as
641 - long doubles (if not equivalent to double) are returned in
642 fpr1,fpr2 for Linux and as for large structs for SysV.
644 - integer values in gpr3;
645 - Structures/Unions by reference;
646 - Single/double FP values in fpr1, long double in fpr1,fpr2.
647 - soft-float float/doubles are treated as UINT32/UINT64 respectivley.
648 - soft-float long doubles are returned in gpr3-gpr6. */
649 /* First translate for softfloat/nonlinux */
650 if (cif
->abi
== FFI_LINUX_SOFT_FLOAT
) {
651 if (type
== FFI_TYPE_FLOAT
)
652 type
= FFI_TYPE_UINT32
;
653 if (type
== FFI_TYPE_DOUBLE
)
654 type
= FFI_TYPE_UINT64
;
655 if (type
== FFI_TYPE_LONGDOUBLE
)
656 type
= FFI_TYPE_UINT128
;
657 } else if (cif
->abi
!= FFI_LINUX
&& cif
->abi
!= FFI_LINUX64
) {
658 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
659 if (type
== FFI_TYPE_LONGDOUBLE
)
660 type
= FFI_TYPE_STRUCT
;
667 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
668 case FFI_TYPE_LONGDOUBLE
:
669 flags
|= FLAG_RETURNS_128BITS
;
672 case FFI_TYPE_DOUBLE
:
673 flags
|= FLAG_RETURNS_64BITS
;
676 flags
|= FLAG_RETURNS_FP
;
680 case FFI_TYPE_UINT128
:
681 flags
|= FLAG_RETURNS_128BITS
;
683 case FFI_TYPE_UINT64
:
684 case FFI_TYPE_SINT64
:
685 flags
|= FLAG_RETURNS_64BITS
;
688 case FFI_TYPE_STRUCT
:
690 * The final SYSV ABI says that structures smaller or equal 8 bytes
691 * are returned in r3/r4. The FFI_GCC_SYSV ABI instead returns them
694 * NOTE: The assembly code can safely assume that it just needs to
695 * store both r3 and r4 into a 8-byte word-aligned buffer, as
696 * we allocate a temporary buffer in ffi_call() if this flag is
699 if (cif
->abi
== FFI_SYSV
&& size
<= 8)
700 flags
|= FLAG_RETURNS_SMST
;
702 flags
|= FLAG_RETVAL_REFERENCE
;
705 flags
|= FLAG_RETURNS_NOTHING
;
709 /* Returns 32-bit integer, or similar. Nothing to do here. */
713 if (cif
->abi
!= FFI_LINUX64
)
714 /* The first NUM_GPR_ARG_REGISTERS words of integer arguments, and the
715 first NUM_FPR_ARG_REGISTERS fp arguments, go in registers; the rest
716 goes on the stack. Structures and long doubles (if not equivalent
717 to double) are passed as a pointer to a copy of the structure.
718 Stuff on the stack needs to keep proper alignment. */
719 for (ptr
= cif
->arg_types
, i
= cif
->nargs
; i
> 0; i
--, ptr
++)
721 unsigned short typenum
= (*ptr
)->type
;
723 /* We may need to handle some values depending on ABI */
724 if (cif
->abi
== FFI_LINUX_SOFT_FLOAT
) {
725 if (typenum
== FFI_TYPE_FLOAT
)
726 typenum
= FFI_TYPE_UINT32
;
727 if (typenum
== FFI_TYPE_DOUBLE
)
728 typenum
= FFI_TYPE_UINT64
;
729 if (typenum
== FFI_TYPE_LONGDOUBLE
)
730 typenum
= FFI_TYPE_UINT128
;
731 } else if (cif
->abi
!= FFI_LINUX
&& cif
->abi
!= FFI_LINUX64
) {
732 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
733 if (typenum
== FFI_TYPE_LONGDOUBLE
)
734 typenum
= FFI_TYPE_STRUCT
;
742 /* floating singles are not 8-aligned on stack */
745 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
746 case FFI_TYPE_LONGDOUBLE
:
750 case FFI_TYPE_DOUBLE
:
752 /* If this FP arg is going on the stack, it must be
754 if (fparg_count
> NUM_FPR_ARG_REGISTERS
755 && intarg_count
>= NUM_GPR_ARG_REGISTERS
756 && intarg_count
% 2 != 0)
760 case FFI_TYPE_UINT128
:
762 * A long double in FFI_LINUX_SOFT_FLOAT can use only a set
763 * of four consecutive gprs. If we do not have enough, we
764 * have to adjust the intarg_count value.
766 if (intarg_count
>= NUM_GPR_ARG_REGISTERS
- 3
767 && intarg_count
< NUM_GPR_ARG_REGISTERS
)
768 intarg_count
= NUM_GPR_ARG_REGISTERS
;
772 case FFI_TYPE_UINT64
:
773 case FFI_TYPE_SINT64
:
774 /* 'long long' arguments are passed as two words, but
775 either both words must fit in registers or both go
776 on the stack. If they go on the stack, they must
779 Also, only certain register pairs can be used for
780 passing long long int -- specifically (r3,r4), (r5,r6),
783 if (intarg_count
== NUM_GPR_ARG_REGISTERS
-1
784 || intarg_count
% 2 != 0)
789 case FFI_TYPE_STRUCT
:
790 /* We must allocate space for a copy of these to enforce
791 pass-by-value. Pad the space up to a multiple of 16
792 bytes (the maximum alignment required for anything under
794 struct_copy_size
+= ((*ptr
)->size
+ 15) & ~0xF;
795 /* Fall through (allocate space for the pointer). */
797 case FFI_TYPE_POINTER
:
799 case FFI_TYPE_UINT32
:
800 case FFI_TYPE_SINT32
:
801 case FFI_TYPE_UINT16
:
802 case FFI_TYPE_SINT16
:
805 /* Everything else is passed as a 4-byte word in a GPR, either
806 the object itself or a pointer to it. */
814 for (ptr
= cif
->arg_types
, i
= cif
->nargs
; i
> 0; i
--, ptr
++)
816 switch ((*ptr
)->type
)
818 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
819 case FFI_TYPE_LONGDOUBLE
:
820 if (cif
->abi
== FFI_LINUX_SOFT_FLOAT
)
830 case FFI_TYPE_DOUBLE
:
835 case FFI_TYPE_STRUCT
:
836 intarg_count
+= ((*ptr
)->size
+ 7) / 8;
839 case FFI_TYPE_POINTER
:
840 case FFI_TYPE_UINT64
:
841 case FFI_TYPE_SINT64
:
843 case FFI_TYPE_UINT32
:
844 case FFI_TYPE_SINT32
:
845 case FFI_TYPE_UINT16
:
846 case FFI_TYPE_SINT16
:
849 /* Everything else is passed as a 8-byte word in a GPR, either
850 the object itself or a pointer to it. */
859 if (fparg_count
!= 0)
860 flags
|= FLAG_FP_ARGUMENTS
;
862 if (intarg_count
> 4)
863 flags
|= FLAG_4_GPR_ARGUMENTS
;
864 if (struct_copy_size
!= 0)
865 flags
|= FLAG_ARG_NEEDS_COPY
;
867 if (cif
->abi
!= FFI_LINUX64
)
870 /* Space for the FPR registers, if needed. */
871 if (fparg_count
!= 0)
872 bytes
+= NUM_FPR_ARG_REGISTERS
* sizeof (double);
876 if (intarg_count
> NUM_GPR_ARG_REGISTERS
)
877 bytes
+= (intarg_count
- NUM_GPR_ARG_REGISTERS
) * sizeof (int);
879 if (fparg_count
> NUM_FPR_ARG_REGISTERS
)
880 bytes
+= (fparg_count
- NUM_FPR_ARG_REGISTERS
) * sizeof (double);
886 /* Space for the FPR registers, if needed. */
887 if (fparg_count
!= 0)
888 bytes
+= NUM_FPR_ARG_REGISTERS64
* sizeof (double);
892 if (intarg_count
> NUM_GPR_ARG_REGISTERS64
)
893 bytes
+= (intarg_count
- NUM_GPR_ARG_REGISTERS64
) * sizeof (long);
896 /* The stack space allocated needs to be a multiple of 16 bytes. */
897 bytes
= (bytes
+ 15) & ~0xF;
899 /* Add in the space for the copied structures. */
900 bytes
+= struct_copy_size
;
908 extern void ffi_call_SYSV(extended_cif
*, unsigned, unsigned, unsigned *,
910 extern void FFI_HIDDEN
ffi_call_LINUX64(extended_cif
*, unsigned long,
911 unsigned long, unsigned long *,
915 ffi_call(ffi_cif
*cif
, void (*fn
)(void), void *rvalue
, void **avalue
)
918 * The final SYSV ABI says that structures smaller or equal 8 bytes
919 * are returned in r3/r4. The FFI_GCC_SYSV ABI instead returns them
922 * Just to keep things simple for the assembly code, we will always
923 * bounce-buffer struct return values less than or equal to 8 bytes.
924 * This allows the ASM to handle SYSV small structures by directly
925 * writing r3 and r4 to memory without worrying about struct size.
927 unsigned int smst_buffer
[2];
929 unsigned int rsize
= 0;
932 ecif
.avalue
= avalue
;
934 /* Ensure that we have a valid struct return value */
935 ecif
.rvalue
= rvalue
;
936 if (cif
->rtype
->type
== FFI_TYPE_STRUCT
) {
937 rsize
= cif
->rtype
->size
;
939 ecif
.rvalue
= smst_buffer
;
941 ecif
.rvalue
= alloca(rsize
);
952 case FFI_LINUX_SOFT_FLOAT
:
953 ffi_call_SYSV (&ecif
, -cif
->bytes
, cif
->flags
, ecif
.rvalue
, fn
);
957 ffi_call_LINUX64 (&ecif
, -(long) cif
->bytes
, cif
->flags
, ecif
.rvalue
, fn
);
965 /* Check for a bounce-buffered return value */
966 if (rvalue
&& ecif
.rvalue
== smst_buffer
)
967 memcpy(rvalue
, smst_buffer
, rsize
);
972 #define MIN_CACHE_LINE_SIZE 8
975 flush_icache (char *wraddr
, char *xaddr
, int size
)
978 for (i
= 0; i
< size
; i
+= MIN_CACHE_LINE_SIZE
)
979 __asm__
volatile ("icbi 0,%0;" "dcbf 0,%1;"
980 : : "r" (xaddr
+ i
), "r" (wraddr
+ i
) : "memory");
981 __asm__
volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
982 : : "r"(xaddr
+ size
- 1), "r"(wraddr
+ size
- 1)
988 ffi_prep_closure_loc (ffi_closure
*closure
,
990 void (*fun
) (ffi_cif
*, void *, void **, void *),
995 void **tramp
= (void **) &closure
->tramp
[0];
997 if (cif
->abi
!= FFI_LINUX64
)
999 /* Copy function address and TOC from ffi_closure_LINUX64. */
1000 memcpy (tramp
, (char *) ffi_closure_LINUX64
, 16);
1003 unsigned int *tramp
;
1005 if (! (cif
->abi
== FFI_GCC_SYSV
1006 || cif
->abi
== FFI_SYSV
1007 || cif
->abi
== FFI_LINUX
1008 || cif
->abi
== FFI_LINUX_SOFT_FLOAT
))
1011 tramp
= (unsigned int *) &closure
->tramp
[0];
1012 tramp
[0] = 0x7c0802a6; /* mflr r0 */
1013 tramp
[1] = 0x4800000d; /* bl 10 <trampoline_initial+0x10> */
1014 tramp
[4] = 0x7d6802a6; /* mflr r11 */
1015 tramp
[5] = 0x7c0803a6; /* mtlr r0 */
1016 tramp
[6] = 0x800b0000; /* lwz r0,0(r11) */
1017 tramp
[7] = 0x816b0004; /* lwz r11,4(r11) */
1018 tramp
[8] = 0x7c0903a6; /* mtctr r0 */
1019 tramp
[9] = 0x4e800420; /* bctr */
1020 *(void **) &tramp
[2] = (void *) ffi_closure_SYSV
; /* function */
1021 *(void **) &tramp
[3] = codeloc
; /* context */
1023 /* Flush the icache. */
1024 flush_icache ((char *)tramp
, (char *)codeloc
, FFI_TRAMPOLINE_SIZE
);
1029 closure
->user_data
= user_data
;
1040 int ffi_closure_helper_SYSV (ffi_closure
*, void *, unsigned long *,
1041 ffi_dblfl
*, unsigned long *);
1043 /* Basically the trampoline invokes ffi_closure_SYSV, and on
1044 * entry, r11 holds the address of the closure.
1045 * After storing the registers that could possibly contain
1046 * parameters to be passed into the stack frame and setting
1047 * up space for a return value, ffi_closure_SYSV invokes the
1048 * following helper function to do most of the work
1052 ffi_closure_helper_SYSV (ffi_closure
*closure
, void *rvalue
,
1053 unsigned long *pgr
, ffi_dblfl
*pfr
,
1056 /* rvalue is the pointer to space for return value in closure assembly */
1057 /* pgr is the pointer to where r3-r10 are stored in ffi_closure_SYSV */
1058 /* pfr is the pointer to where f1-f8 are stored in ffi_closure_SYSV */
1059 /* pst is the pointer to outgoing parameter stack in original caller */
1062 ffi_type
** arg_types
;
1065 long nf
= 0; /* number of floating registers already used */
1067 long ng
= 0; /* number of general registers already used */
1069 ffi_cif
*cif
= closure
->cif
;
1070 unsigned size
= cif
->rtype
->size
;
1071 unsigned short rtypenum
= cif
->rtype
->type
;
1073 avalue
= alloca (cif
->nargs
* sizeof (void *));
1075 /* First translate for softfloat/nonlinux */
1076 if (cif
->abi
== FFI_LINUX_SOFT_FLOAT
) {
1077 if (rtypenum
== FFI_TYPE_FLOAT
)
1078 rtypenum
= FFI_TYPE_UINT32
;
1079 if (rtypenum
== FFI_TYPE_DOUBLE
)
1080 rtypenum
= FFI_TYPE_UINT64
;
1081 if (rtypenum
== FFI_TYPE_LONGDOUBLE
)
1082 rtypenum
= FFI_TYPE_UINT128
;
1083 } else if (cif
->abi
!= FFI_LINUX
&& cif
->abi
!= FFI_LINUX64
) {
1084 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1085 if (rtypenum
== FFI_TYPE_LONGDOUBLE
)
1086 rtypenum
= FFI_TYPE_STRUCT
;
1091 /* Copy the caller's structure return value address so that the closure
1092 returns the data directly to the caller.
1093 For FFI_SYSV the result is passed in r3/r4 if the struct size is less
1094 or equal 8 bytes. */
1095 if (rtypenum
== FFI_TYPE_STRUCT
&& ((cif
->abi
!= FFI_SYSV
) || (size
> 8))) {
1096 rvalue
= (void *) *pgr
;
1103 arg_types
= cif
->arg_types
;
1105 /* Grab the addresses of the arguments from the stack frame. */
1107 unsigned short typenum
= arg_types
[i
]->type
;
1109 /* We may need to handle some values depending on ABI */
1110 if (cif
->abi
== FFI_LINUX_SOFT_FLOAT
) {
1111 if (typenum
== FFI_TYPE_FLOAT
)
1112 typenum
= FFI_TYPE_UINT32
;
1113 if (typenum
== FFI_TYPE_DOUBLE
)
1114 typenum
= FFI_TYPE_UINT64
;
1115 if (typenum
== FFI_TYPE_LONGDOUBLE
)
1116 typenum
= FFI_TYPE_UINT128
;
1117 } else if (cif
->abi
!= FFI_LINUX
&& cif
->abi
!= FFI_LINUX64
) {
1118 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1119 if (typenum
== FFI_TYPE_LONGDOUBLE
)
1120 typenum
= FFI_TYPE_STRUCT
;
1126 case FFI_TYPE_FLOAT
:
1127 /* unfortunately float values are stored as doubles
1128 * in the ffi_closure_SYSV code (since we don't check
1129 * the type in that routine).
1132 /* there are 8 64bit floating point registers */
1136 double temp
= pfr
->d
;
1137 pfr
->f
= (float) temp
;
1144 /* FIXME? here we are really changing the values
1145 * stored in the original calling routines outgoing
1146 * parameter stack. This is probably a really
1147 * naughty thing to do but...
1154 case FFI_TYPE_DOUBLE
:
1155 /* On the outgoing stack all values are aligned to 8 */
1156 /* there are 8 64bit floating point registers */
1166 if (((long) pst
) & 4)
1173 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1174 case FFI_TYPE_LONGDOUBLE
:
1183 if (((long) pst
) & 4)
1191 #endif /* have FPRS */
1193 case FFI_TYPE_UINT128
:
1195 * Test if for the whole long double, 4 gprs are available.
1196 * otherwise the stuff ends up on the stack.
1209 case FFI_TYPE_SINT8
:
1210 case FFI_TYPE_UINT8
:
1211 /* there are 8 gpr registers used to pass values */
1214 avalue
[i
] = (char *) pgr
+ 3;
1220 avalue
[i
] = (char *) pst
+ 3;
1225 case FFI_TYPE_SINT16
:
1226 case FFI_TYPE_UINT16
:
1227 /* there are 8 gpr registers used to pass values */
1230 avalue
[i
] = (char *) pgr
+ 2;
1236 avalue
[i
] = (char *) pst
+ 2;
1241 case FFI_TYPE_SINT32
:
1242 case FFI_TYPE_UINT32
:
1243 case FFI_TYPE_POINTER
:
1244 /* there are 8 gpr registers used to pass values */
1258 case FFI_TYPE_STRUCT
:
1259 /* Structs are passed by reference. The address will appear in a
1260 gpr if it is one of the first 8 arguments. */
1263 avalue
[i
] = (void *) *pgr
;
1269 avalue
[i
] = (void *) *pst
;
1274 case FFI_TYPE_SINT64
:
1275 case FFI_TYPE_UINT64
:
1276 /* passing long long ints are complex, they must
1277 * be passed in suitable register pairs such as
1278 * (r3,r4) or (r5,r6) or (r6,r7), or (r7,r8) or (r9,r10)
1279 * and if the entire pair aren't available then the outgoing
1280 * parameter stack is used for both but an alignment of 8
1281 * must will be kept. So we must either look in pgr
1282 * or pst to find the correct address for this type
1289 /* skip r4, r6, r8 as starting points */
1299 if (((long) pst
) & 4)
1315 (closure
->fun
) (cif
, rvalue
, avalue
, closure
->user_data
);
1317 /* Tell ffi_closure_SYSV how to perform return type promotions.
1318 Because the FFI_SYSV ABI returns the structures <= 8 bytes in r3/r4
1319 we have to tell ffi_closure_SYSV how to treat them. We combine the base
1320 type FFI_SYSV_TYPE_SMALL_STRUCT - 1 with the size of the struct.
1321 So a one byte struct gets the return type 16. Return type 1 to 15 are
1322 already used and we never have a struct with size zero. That is the reason
1323 for the subtraction of 1. See the comment in ffitarget.h about ordering.
1325 if (cif
->abi
== FFI_SYSV
&& rtypenum
== FFI_TYPE_STRUCT
&& size
<= 8)
1326 return (FFI_SYSV_TYPE_SMALL_STRUCT
- 1) + size
;
1330 int FFI_HIDDEN
ffi_closure_helper_LINUX64 (ffi_closure
*, void *,
1331 unsigned long *, ffi_dblfl
*);
1334 ffi_closure_helper_LINUX64 (ffi_closure
*closure
, void *rvalue
,
1335 unsigned long *pst
, ffi_dblfl
*pfr
)
1337 /* rvalue is the pointer to space for return value in closure assembly */
1338 /* pst is the pointer to parameter save area
1339 (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
1340 /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
1343 ffi_type
**arg_types
;
1346 ffi_dblfl
*end_pfr
= pfr
+ NUM_FPR_ARG_REGISTERS64
;
1349 avalue
= alloca (cif
->nargs
* sizeof (void *));
1351 /* Copy the caller's structure return value address so that the closure
1352 returns the data directly to the caller. */
1353 if (cif
->rtype
->type
== FFI_TYPE_STRUCT
)
1355 rvalue
= (void *) *pst
;
1361 arg_types
= cif
->arg_types
;
1363 /* Grab the addresses of the arguments from the stack frame. */
1366 switch (arg_types
[i
]->type
)
1368 case FFI_TYPE_SINT8
:
1369 case FFI_TYPE_UINT8
:
1370 avalue
[i
] = (char *) pst
+ 7;
1374 case FFI_TYPE_SINT16
:
1375 case FFI_TYPE_UINT16
:
1376 avalue
[i
] = (char *) pst
+ 6;
1380 case FFI_TYPE_SINT32
:
1381 case FFI_TYPE_UINT32
:
1382 avalue
[i
] = (char *) pst
+ 4;
1386 case FFI_TYPE_SINT64
:
1387 case FFI_TYPE_UINT64
:
1388 case FFI_TYPE_POINTER
:
1393 case FFI_TYPE_STRUCT
:
1394 /* Structures with size less than eight bytes are passed
1396 if (arg_types
[i
]->size
< 8)
1397 avalue
[i
] = (char *) pst
+ 8 - arg_types
[i
]->size
;
1400 pst
+= (arg_types
[i
]->size
+ 7) / 8;
1403 case FFI_TYPE_FLOAT
:
1404 /* unfortunately float values are stored as doubles
1405 * in the ffi_closure_LINUX64 code (since we don't check
1406 * the type in that routine).
1409 /* there are 13 64bit floating point registers */
1413 double temp
= pfr
->d
;
1414 pfr
->f
= (float) temp
;
1423 case FFI_TYPE_DOUBLE
:
1424 /* On the outgoing stack all values are aligned to 8 */
1425 /* there are 13 64bit floating point registers */
1437 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1438 case FFI_TYPE_LONGDOUBLE
:
1439 if (pfr
+ 1 < end_pfr
)
1448 /* Passed partly in f13 and partly on the stack.
1449 Move it all to the stack. */
1450 *pst
= *(unsigned long *) pfr
;
1467 (closure
->fun
) (cif
, rvalue
, avalue
, closure
->user_data
);
1469 /* Tell ffi_closure_LINUX64 how to perform return type promotions. */
1470 return cif
->rtype
->type
;