libffi/src/x86/ffi64.c

   1 /* -----------------------------------------------------------------------
   2    ffi64.c - Copyright (c) 2011, 2018  Anthony Green
   3              Copyright (c) 2013  The Written Word, Inc.
   4              Copyright (c) 2008, 2010  Red Hat, Inc.
   5              Copyright (c) 2002, 2007  Bo Thorsen <bo@suse.de>
   6
   7    x86-64 Foreign Function Interface
   8
   9    Permission is hereby granted, free of charge, to any person obtaining
  10    a copy of this software and associated documentation files (the
  11    ``Software''), to deal in the Software without restriction, including
  12    without limitation the rights to use, copy, modify, merge, publish,
  13    distribute, sublicense, and/or sell copies of the Software, and to
  14    permit persons to whom the Software is furnished to do so, subject to
  15    the following conditions:
  16
  17    The above copyright notice and this permission notice shall be included
  18    in all copies or substantial portions of the Software.
  19
  20    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  21    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23    NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  24    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  25    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  26    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  27    DEALINGS IN THE SOFTWARE.
  28    ----------------------------------------------------------------------- */
  29
  30 #include <ffi.h>
  31 #include <ffi_common.h>
  32
  33 #include <stdlib.h>
  34 #include <stdarg.h>
  35 #include <stdint.h>
  36 #include <tramp.h>
  37 #include "internal64.h"
  38
  39 #ifdef __x86_64__
  40
  41 #define MAX_GPR_REGS 6
  42 #define MAX_SSE_REGS 8
  43
  44 #if defined(__INTEL_COMPILER)
  45 #include "xmmintrin.h"
  46 #define UINT128 __m128
  47 #else
  48 #if defined(__SUNPRO_C)
  49 #include <sunmedia_types.h>
  50 #define UINT128 __m128i
  51 #else
  52 #define UINT128 __int128_t
  53 #endif
  54 #endif
  55
  56 union big_int_union
  57 {
  58   UINT32 i32;
  59   UINT64 i64;
  60   UINT128 i128;
  61 };
  62
  63 struct register_args
  64 {
  65   /* Registers for argument passing.  */
  66   UINT64 gpr[MAX_GPR_REGS];
  67   union big_int_union sse[MAX_SSE_REGS];
  68   UINT64 rax;   /* ssecount */
  69   UINT64 r10;   /* static chain */
  70 };
  71
  72 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
  73                              void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
  74
  75 /* All reference to register classes here is identical to the code in
  76    gcc/config/i386/i386.c. Do *not* change one without the other.  */
  77
  78 /* Register class used for passing given 64bit part of the argument.
  79    These represent classes as documented by the PS ABI, with the
  80    exception of SSESF, SSEDF classes, that are basically SSE class,
  81    just gcc will use SF or DFmode move instead of DImode to avoid
  82    reformatting penalties.
  83
  84    Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
  85    whenever possible (upper half does contain padding).  */
  86 enum x86_64_reg_class
  87   {
  88     X86_64_NO_CLASS,
  89     X86_64_INTEGER_CLASS,
  90     X86_64_INTEGERSI_CLASS,
  91     X86_64_SSE_CLASS,
  92     X86_64_SSESF_CLASS,
  93     X86_64_SSEDF_CLASS,
  94     X86_64_SSEUP_CLASS,
  95     X86_64_X87_CLASS,
  96     X86_64_X87UP_CLASS,
  97     X86_64_COMPLEX_X87_CLASS,
  98     X86_64_MEMORY_CLASS
  99   };
 100
 101 #define MAX_CLASSES 4
 102
 103 #define SSE_CLASS_P(X)  ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
 104
 105 /* x86-64 register passing implementation.  See x86-64 ABI for details.  Goal
 106    of this code is to classify each 8bytes of incoming argument by the register
 107    class and assign registers accordingly.  */
 108
 109 /* Return the union class of CLASS1 and CLASS2.
 110    See the x86-64 PS ABI for details.  */
 111
 112 static enum x86_64_reg_class
 113 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
 114 {
 115   /* Rule #1: If both classes are equal, this is the resulting class.  */
 116   if (class1 == class2)
 117     return class1;
 118
 119   /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
 120      the other class.  */
 121   if (class1 == X86_64_NO_CLASS)
 122     return class2;
 123   if (class2 == X86_64_NO_CLASS)
 124     return class1;
 125
 126   /* Rule #3: If one of the classes is MEMORY, the result is MEMORY.  */
 127   if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
 128     return X86_64_MEMORY_CLASS;
 129
 130   /* Rule #4: If one of the classes is INTEGER, the result is INTEGER.  */
 131   if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
 132       || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
 133     return X86_64_INTEGERSI_CLASS;
 134   if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
 135       || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
 136     return X86_64_INTEGER_CLASS;
 137
 138   /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
 139      MEMORY is used.  */
 140   if (class1 == X86_64_X87_CLASS
 141       || class1 == X86_64_X87UP_CLASS
 142       || class1 == X86_64_COMPLEX_X87_CLASS
 143       || class2 == X86_64_X87_CLASS
 144       || class2 == X86_64_X87UP_CLASS
 145       || class2 == X86_64_COMPLEX_X87_CLASS)
 146     return X86_64_MEMORY_CLASS;
 147
 148   /* Rule #6: Otherwise class SSE is used.  */
 149   return X86_64_SSE_CLASS;
 150 }
 151
 152 /* Classify the argument of type TYPE and mode MODE.
 153    CLASSES will be filled by the register class used to pass each word
 154    of the operand.  The number of words is returned.  In case the parameter
 155    should be passed in memory, 0 is returned. As a special case for zero
 156    sized containers, classes[0] will be NO_CLASS and 1 is returned.
 157
 158    See the x86-64 PS ABI for details.
 159 */
 160 static size_t
 161 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
 162                    size_t byte_offset)
 163 {
 164   switch (type->type)
 165     {
 166     case FFI_TYPE_UINT8:
 167     case FFI_TYPE_SINT8:
 168     case FFI_TYPE_UINT16:
 169     case FFI_TYPE_SINT16:
 170     case FFI_TYPE_UINT32:
 171     case FFI_TYPE_SINT32:
 172     case FFI_TYPE_UINT64:
 173     case FFI_TYPE_SINT64:
 174     case FFI_TYPE_POINTER:
 175     do_integer:
 176       {
 177         size_t size = byte_offset + type->size;
 178
 179         if (size <= 4)
 180           {
 181             classes[0] = X86_64_INTEGERSI_CLASS;
 182             return 1;
 183           }
 184         else if (size <= 8)
 185           {
 186             classes[0] = X86_64_INTEGER_CLASS;
 187             return 1;
 188           }
 189         else if (size <= 12)
 190           {
 191             classes[0] = X86_64_INTEGER_CLASS;
 192             classes[1] = X86_64_INTEGERSI_CLASS;
 193             return 2;
 194           }
 195         else if (size <= 16)
 196           {
 197             classes[0] = classes[1] = X86_64_INTEGER_CLASS;
 198             return 2;
 199           }
 200         else
 201           FFI_ASSERT (0);
 202       }
 203     case FFI_TYPE_FLOAT:
 204       if (!(byte_offset % 8))
 205         classes[0] = X86_64_SSESF_CLASS;
 206       else
 207         classes[0] = X86_64_SSE_CLASS;
 208       return 1;
 209     case FFI_TYPE_DOUBLE:
 210       classes[0] = X86_64_SSEDF_CLASS;
 211       return 1;
 212 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 213     case FFI_TYPE_LONGDOUBLE:
 214       classes[0] = X86_64_X87_CLASS;
 215       classes[1] = X86_64_X87UP_CLASS;
 216       return 2;
 217 #endif
 218     case FFI_TYPE_STRUCT:
 219       {
 220         const size_t UNITS_PER_WORD = 8;
 221         size_t words = (type->size + byte_offset + UNITS_PER_WORD - 1)
 222                        / UNITS_PER_WORD;
 223         ffi_type **ptr;
 224         unsigned int i;
 225         enum x86_64_reg_class subclasses[MAX_CLASSES];
 226
 227         /* If the struct is larger than 32 bytes, pass it on the stack.  */
 228         if (type->size > 32)
 229           return 0;
 230
 231         for (i = 0; i < words; i++)
 232           classes[i] = X86_64_NO_CLASS;
 233
 234         /* Zero sized arrays or structures are NO_CLASS.  We return 0 to
 235            signalize memory class, so handle it as special case.  */
 236         if (!words)
 237           {
 238     case FFI_TYPE_VOID:
 239             classes[0] = X86_64_NO_CLASS;
 240             return 1;
 241           }
 242
 243         /* Merge the fields of structure.  */
 244         for (ptr = type->elements; *ptr != NULL; ptr++)
 245           {
 246             size_t num, pos;
 247
 248             byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
 249
 250             num = classify_argument (*ptr, subclasses, byte_offset % 8);
 251             if (num == 0)
 252               return 0;
 253             pos = byte_offset / 8;
 254             for (i = 0; i < num && (i + pos) < words; i++)
 255               {
 256                 size_t pos = byte_offset / 8;
 257                 classes[i + pos] =
 258                   merge_classes (subclasses[i], classes[i + pos]);
 259               }
 260
 261             byte_offset += (*ptr)->size;
 262           }
 263
 264         if (words > 2)
 265           {
 266             /* When size > 16 bytes, if the first one isn't
 267                X86_64_SSE_CLASS or any other ones aren't
 268                X86_64_SSEUP_CLASS, everything should be passed in
 269                memory.  */
 270             if (classes[0] != X86_64_SSE_CLASS)
 271               return 0;
 272
 273             for (i = 1; i < words; i++)
 274               if (classes[i] != X86_64_SSEUP_CLASS)
 275                 return 0;
 276           }
 277
 278         /* Final merger cleanup.  */
 279         for (i = 0; i < words; i++)
 280           {
 281             /* If one class is MEMORY, everything should be passed in
 282                memory.  */
 283             if (classes[i] == X86_64_MEMORY_CLASS)
 284               return 0;
 285
 286             /* The X86_64_SSEUP_CLASS should be always preceded by
 287                X86_64_SSE_CLASS or X86_64_SSEUP_CLASS.  */
 288             if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
 289                 && classes[i - 1] != X86_64_SSE_CLASS
 290                 && classes[i - 1] != X86_64_SSEUP_CLASS)
 291               {
 292                 /* The first one should never be X86_64_SSEUP_CLASS.  */
 293                 FFI_ASSERT (i != 0);
 294                 classes[i] = X86_64_SSE_CLASS;
 295               }
 296
 297             /*  If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
 298                 everything should be passed in memory.  */
 299             if (i > 1 && classes[i] == X86_64_X87UP_CLASS
 300                 && (classes[i - 1] != X86_64_X87_CLASS))
 301               {
 302                 /* The first one should never be X86_64_X87UP_CLASS.  */
 303                 FFI_ASSERT (i != 0);
 304                 return 0;
 305               }
 306           }
 307         return words;
 308       }
 309     case FFI_TYPE_COMPLEX:
 310       {
 311         ffi_type *inner = type->elements[0];
 312         switch (inner->type)
 313           {
 314           case FFI_TYPE_INT:
 315           case FFI_TYPE_UINT8:
 316           case FFI_TYPE_SINT8:
 317           case FFI_TYPE_UINT16:
 318           case FFI_TYPE_SINT16:
 319           case FFI_TYPE_UINT32:
 320           case FFI_TYPE_SINT32:
 321           case FFI_TYPE_UINT64:
 322           case FFI_TYPE_SINT64:
 323             goto do_integer;
 324
 325           case FFI_TYPE_FLOAT:
 326             classes[0] = X86_64_SSE_CLASS;
 327             if (byte_offset % 8)
 328               {
 329                 classes[1] = X86_64_SSESF_CLASS;
 330                 return 2;
 331               }
 332             return 1;
 333           case FFI_TYPE_DOUBLE:
 334             classes[0] = classes[1] = X86_64_SSEDF_CLASS;
 335             return 2;
 336 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 337           case FFI_TYPE_LONGDOUBLE:
 338             classes[0] = X86_64_COMPLEX_X87_CLASS;
 339             return 1;
 340 #endif
 341           }
 342       }
 343     }
 344   abort();
 345 }
 346
 347 /* Examine the argument and return set number of register required in each
 348    class.  Return zero iff parameter should be passed in memory, otherwise
 349    the number of registers.  */
 350
 351 static size_t
 352 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
 353                   _Bool in_return, int *pngpr, int *pnsse)
 354 {
 355   size_t n;
 356   unsigned int i;
 357   int ngpr, nsse;
 358
 359   n = classify_argument (type, classes, 0);
 360   if (n == 0)
 361     return 0;
 362
 363   ngpr = nsse = 0;
 364   for (i = 0; i < n; ++i)
 365     switch (classes[i])
 366       {
 367       case X86_64_INTEGER_CLASS:
 368       case X86_64_INTEGERSI_CLASS:
 369         ngpr++;
 370         break;
 371       case X86_64_SSE_CLASS:
 372       case X86_64_SSESF_CLASS:
 373       case X86_64_SSEDF_CLASS:
 374         nsse++;
 375         break;
 376       case X86_64_NO_CLASS:
 377       case X86_64_SSEUP_CLASS:
 378         break;
 379       case X86_64_X87_CLASS:
 380       case X86_64_X87UP_CLASS:
 381       case X86_64_COMPLEX_X87_CLASS:
 382         return in_return != 0;
 383       default:
 384         abort ();
 385       }
 386
 387   *pngpr = ngpr;
 388   *pnsse = nsse;
 389
 390   return n;
 391 }
 392
 393 /* Perform machine dependent cif processing.  */
 394
 395 #ifndef __ILP32__
 396 extern ffi_status
 397 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
 398 #endif
 399
 400 ffi_status FFI_HIDDEN
 401 ffi_prep_cif_machdep (ffi_cif *cif)
 402 {
 403   int gprcount, ssecount, i, avn, ngpr, nsse;
 404   unsigned flags;
 405   enum x86_64_reg_class classes[MAX_CLASSES];
 406   size_t bytes, n, rtype_size;
 407   ffi_type *rtype;
 408
 409 #ifndef __ILP32__
 410   if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
 411     return ffi_prep_cif_machdep_efi64(cif);
 412 #endif
 413   if (cif->abi != FFI_UNIX64)
 414     return FFI_BAD_ABI;
 415
 416   gprcount = ssecount = 0;
 417
 418   rtype = cif->rtype;
 419   rtype_size = rtype->size;
 420   switch (rtype->type)
 421     {
 422     case FFI_TYPE_VOID:
 423       flags = UNIX64_RET_VOID;
 424       break;
 425     case FFI_TYPE_UINT8:
 426       flags = UNIX64_RET_UINT8;
 427       break;
 428     case FFI_TYPE_SINT8:
 429       flags = UNIX64_RET_SINT8;
 430       break;
 431     case FFI_TYPE_UINT16:
 432       flags = UNIX64_RET_UINT16;
 433       break;
 434     case FFI_TYPE_SINT16:
 435       flags = UNIX64_RET_SINT16;
 436       break;
 437     case FFI_TYPE_UINT32:
 438       flags = UNIX64_RET_UINT32;
 439       break;
 440     case FFI_TYPE_INT:
 441     case FFI_TYPE_SINT32:
 442       flags = UNIX64_RET_SINT32;
 443       break;
 444     case FFI_TYPE_UINT64:
 445     case FFI_TYPE_SINT64:
 446       flags = UNIX64_RET_INT64;
 447       break;
 448     case FFI_TYPE_POINTER:
 449       flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
 450       break;
 451     case FFI_TYPE_FLOAT:
 452       flags = UNIX64_RET_XMM32;
 453       break;
 454     case FFI_TYPE_DOUBLE:
 455       flags = UNIX64_RET_XMM64;
 456       break;
 457 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 458     case FFI_TYPE_LONGDOUBLE:
 459       flags = UNIX64_RET_X87;
 460       break;
 461 #endif
 462     case FFI_TYPE_STRUCT:
 463       n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
 464       if (n == 0)
 465         {
 466           /* The return value is passed in memory.  A pointer to that
 467              memory is the first argument.  Allocate a register for it.  */
 468           gprcount++;
 469           /* We don't have to do anything in asm for the return.  */
 470           flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
 471         }
 472       else
 473         {
 474           _Bool sse0 = SSE_CLASS_P (classes[0]);
 475
 476           if (rtype_size == 4 && sse0)
 477             flags = UNIX64_RET_XMM32;
 478           else if (rtype_size == 8)
 479             flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
 480           else
 481             {
 482               _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
 483               if (sse0 && sse1)
 484                 flags = UNIX64_RET_ST_XMM0_XMM1;
 485               else if (sse0)
 486                 flags = UNIX64_RET_ST_XMM0_RAX;
 487               else if (sse1)
 488                 flags = UNIX64_RET_ST_RAX_XMM0;
 489               else
 490                 flags = UNIX64_RET_ST_RAX_RDX;
 491               flags |= rtype_size << UNIX64_SIZE_SHIFT;
 492             }
 493         }
 494       break;
 495     case FFI_TYPE_COMPLEX:
 496       switch (rtype->elements[0]->type)
 497         {
 498         case FFI_TYPE_UINT8:
 499         case FFI_TYPE_SINT8:
 500         case FFI_TYPE_UINT16:
 501         case FFI_TYPE_SINT16:
 502         case FFI_TYPE_INT:
 503         case FFI_TYPE_UINT32:
 504         case FFI_TYPE_SINT32:
 505         case FFI_TYPE_UINT64:
 506         case FFI_TYPE_SINT64:
 507           flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
 508           break;
 509         case FFI_TYPE_FLOAT:
 510           flags = UNIX64_RET_XMM64;
 511           break;
 512         case FFI_TYPE_DOUBLE:
 513           flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
 514           break;
 515 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 516         case FFI_TYPE_LONGDOUBLE:
 517           flags = UNIX64_RET_X87_2;
 518           break;
 519 #endif
 520         default:
 521           return FFI_BAD_TYPEDEF;
 522         }
 523       break;
 524     default:
 525       return FFI_BAD_TYPEDEF;
 526     }
 527
 528   /* Go over all arguments and determine the way they should be passed.
 529      If it's in a register and there is space for it, let that be so. If
 530      not, add it's size to the stack byte count.  */
 531   for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
 532     {
 533       if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
 534           || gprcount + ngpr > MAX_GPR_REGS
 535           || ssecount + nsse > MAX_SSE_REGS)
 536         {
 537           long align = cif->arg_types[i]->alignment;
 538
 539           if (align < 8)
 540             align = 8;
 541
 542           bytes = FFI_ALIGN (bytes, align);
 543           bytes += cif->arg_types[i]->size;
 544         }
 545       else
 546         {
 547           gprcount += ngpr;
 548           ssecount += nsse;
 549         }
 550     }
 551   if (ssecount)
 552     flags |= UNIX64_FLAG_XMM_ARGS;
 553
 554   cif->flags = flags;
 555   cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
 556
 557   return FFI_OK;
 558 }
 559
 560 static void
 561 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
 562               void **avalue, void *closure)
 563 {
 564   enum x86_64_reg_class classes[MAX_CLASSES];
 565   char *stack, *argp;
 566   ffi_type **arg_types;
 567   int gprcount, ssecount, ngpr, nsse, i, avn, flags;
 568   struct register_args *reg_args;
 569
 570   /* Can't call 32-bit mode from 64-bit mode.  */
 571   FFI_ASSERT (cif->abi == FFI_UNIX64);
 572
 573   /* If the return value is a struct and we don't have a return value
 574      address then we need to make one.  Otherwise we can ignore it.  */
 575   flags = cif->flags;
 576   if (rvalue == NULL)
 577     {
 578       if (flags & UNIX64_FLAG_RET_IN_MEM)
 579         rvalue = alloca (cif->rtype->size);
 580       else
 581         flags = UNIX64_RET_VOID;
 582     }
 583
 584   /* Allocate the space for the arguments, plus 4 words of temp space.  */
 585   stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
 586   reg_args = (struct register_args *) stack;
 587   argp = stack + sizeof (struct register_args);
 588
 589   reg_args->r10 = (uintptr_t) closure;
 590
 591   gprcount = ssecount = 0;
 592
 593   /* If the return value is passed in memory, add the pointer as the
 594      first integer argument.  */
 595   if (flags & UNIX64_FLAG_RET_IN_MEM)
 596     reg_args->gpr[gprcount++] = (unsigned long) rvalue;
 597
 598   avn = cif->nargs;
 599   arg_types = cif->arg_types;
 600
 601   for (i = 0; i < avn; ++i)
 602     {
 603       size_t n, size = arg_types[i]->size;
 604
 605       n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
 606       if (n == 0
 607           || gprcount + ngpr > MAX_GPR_REGS
 608           || ssecount + nsse > MAX_SSE_REGS)
 609         {
 610           long align = arg_types[i]->alignment;
 611
 612           /* Stack arguments are *always* at least 8 byte aligned.  */
 613           if (align < 8)
 614             align = 8;
 615
 616           /* Pass this argument in memory.  */
 617           argp = (void *) FFI_ALIGN (argp, align);
 618           memcpy (argp, avalue[i], size);
 619           argp += size;
 620         }
 621       else
 622         {
 623           /* The argument is passed entirely in registers.  */
 624           char *a = (char *) avalue[i];
 625           unsigned int j;
 626
 627           for (j = 0; j < n; j++, a += 8, size -= 8)
 628             {
 629               switch (classes[j])
 630                 {
 631                 case X86_64_NO_CLASS:
 632                 case X86_64_SSEUP_CLASS:
 633                   break;
 634                 case X86_64_INTEGER_CLASS:
 635                 case X86_64_INTEGERSI_CLASS:
 636                   /* Sign-extend integer arguments passed in general
 637                      purpose registers, to cope with the fact that
 638                      LLVM incorrectly assumes that this will be done
 639                      (the x86-64 PS ABI does not specify this). */
 640                   switch (arg_types[i]->type)
 641                     {
 642                     case FFI_TYPE_SINT8:
 643                       reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
 644                       break;
 645                     case FFI_TYPE_SINT16:
 646                       reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
 647                       break;
 648                     case FFI_TYPE_SINT32:
 649                       reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
 650                       break;
 651                     default:
 652                       reg_args->gpr[gprcount] = 0;
 653                       memcpy (&reg_args->gpr[gprcount], a, size);
 654                     }
 655                   gprcount++;
 656                   break;
 657                 case X86_64_SSE_CLASS:
 658                 case X86_64_SSEDF_CLASS:
 659                   memcpy (&reg_args->sse[ssecount++].i64, a, sizeof(UINT64));
 660                   break;
 661                 case X86_64_SSESF_CLASS:
 662                   memcpy (&reg_args->sse[ssecount++].i32, a, sizeof(UINT32));
 663                   break;
 664                 default:
 665                   abort();
 666                 }
 667             }
 668         }
 669     }
 670   reg_args->rax = ssecount;
 671
 672   ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
 673                    flags, rvalue, fn);
 674 }
 675
 676 #ifndef __ILP32__
 677 extern void
 678 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
 679 #endif
 680
 681 void
 682 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
 683 {
 684 #ifndef __ILP32__
 685   if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
 686     {
 687       ffi_call_efi64(cif, fn, rvalue, avalue);
 688       return;
 689     }
 690 #endif
 691   ffi_call_int (cif, fn, rvalue, avalue, NULL);
 692 }
 693
 694 #ifdef FFI_GO_CLOSURES
 695
 696 #ifndef __ILP32__
 697 extern void
 698 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
 699                   void **avalue, void *closure);
 700 #endif
 701
 702 void
 703 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
 704              void **avalue, void *closure)
 705 {
 706 #ifndef __ILP32__
 707   if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
 708     {
 709       ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
 710       return;
 711     }
 712 #endif
 713   ffi_call_int (cif, fn, rvalue, avalue, closure);
 714 }
 715
 716 #endif /* FFI_GO_CLOSURES */
 717
 718 extern void ffi_closure_unix64(void) FFI_HIDDEN;
 719 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
 720 #if defined(FFI_EXEC_STATIC_TRAMP)
 721 extern void ffi_closure_unix64_alt(void) FFI_HIDDEN;
 722 extern void ffi_closure_unix64_sse_alt(void) FFI_HIDDEN;
 723 #endif
 724
 725 #ifndef __ILP32__
 726 extern ffi_status
 727 ffi_prep_closure_loc_efi64(ffi_closure* closure,
 728                            ffi_cif* cif,
 729                            void (*fun)(ffi_cif*, void*, void**, void*),
 730                            void *user_data,
 731                            void *codeloc);
 732 #endif
 733
 734 ffi_status
 735 ffi_prep_closure_loc (ffi_closure* closure,
 736                       ffi_cif* cif,
 737                       void (*fun)(ffi_cif*, void*, void**, void*),
 738                       void *user_data,
 739                       void *codeloc)
 740 {
 741   static const unsigned char trampoline[24] = {
 742     /* endbr64 */
 743     0xf3, 0x0f, 0x1e, 0xfa,
 744     /* leaq  -0xb(%rip),%r10   # 0x0  */
 745     0x4c, 0x8d, 0x15, 0xf5, 0xff, 0xff, 0xff,
 746     /* jmpq  *0x7(%rip)        # 0x18 */
 747     0xff, 0x25, 0x07, 0x00, 0x00, 0x00,
 748     /* nopl  0(%rax) */
 749     0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00
 750   };
 751   void (*dest)(void);
 752   char *tramp = closure->tramp;
 753
 754 #ifndef __ILP32__
 755   if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
 756     return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
 757 #endif
 758   if (cif->abi != FFI_UNIX64)
 759     return FFI_BAD_ABI;
 760
 761   if (cif->flags & UNIX64_FLAG_XMM_ARGS)
 762     dest = ffi_closure_unix64_sse;
 763   else
 764     dest = ffi_closure_unix64;
 765
 766 #if defined(FFI_EXEC_STATIC_TRAMP)
 767   if (ffi_tramp_is_present(closure))
 768     {
 769       /* Initialize the static trampoline's parameters. */
 770       if (dest == ffi_closure_unix64_sse)
 771         dest = ffi_closure_unix64_sse_alt;
 772       else
 773         dest = ffi_closure_unix64_alt;
 774       ffi_tramp_set_parms (closure->ftramp, dest, closure);
 775       goto out;
 776     }
 777 #endif
 778
 779   /* Initialize the dynamic trampoline. */
 780   memcpy (tramp, trampoline, sizeof(trampoline));
 781   *(UINT64 *)(tramp + sizeof (trampoline)) = (uintptr_t)dest;
 782
 783 out:
 784   closure->cif = cif;
 785   closure->fun = fun;
 786   closure->user_data = user_data;
 787
 788   return FFI_OK;
 789 }
 790
 791 int FFI_HIDDEN
 792 ffi_closure_unix64_inner(ffi_cif *cif,
 793                          void (*fun)(ffi_cif*, void*, void**, void*),
 794                          void *user_data,
 795                          void *rvalue,
 796                          struct register_args *reg_args,
 797                          char *argp)
 798 {
 799   void **avalue;
 800   ffi_type **arg_types;
 801   long i, avn;
 802   int gprcount, ssecount, ngpr, nsse;
 803   int flags;
 804
 805   avn = cif->nargs;
 806   flags = cif->flags;
 807   avalue = alloca(avn * sizeof(void *));
 808   gprcount = ssecount = 0;
 809
 810   if (flags & UNIX64_FLAG_RET_IN_MEM)
 811     {
 812       /* On return, %rax will contain the address that was passed
 813          by the caller in %rdi.  */
 814       void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
 815       *(void **)rvalue = r;
 816       rvalue = r;
 817       flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
 818     }
 819
 820   arg_types = cif->arg_types;
 821   for (i = 0; i < avn; ++i)
 822     {
 823       enum x86_64_reg_class classes[MAX_CLASSES];
 824       size_t n;
 825
 826       n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
 827       if (n == 0
 828           || gprcount + ngpr > MAX_GPR_REGS
 829           || ssecount + nsse > MAX_SSE_REGS)
 830         {
 831           long align = arg_types[i]->alignment;
 832
 833           /* Stack arguments are *always* at least 8 byte aligned.  */
 834           if (align < 8)
 835             align = 8;
 836
 837           /* Pass this argument in memory.  */
 838           argp = (void *) FFI_ALIGN (argp, align);
 839           avalue[i] = argp;
 840           argp += arg_types[i]->size;
 841         }
 842       /* If the argument is in a single register, or two consecutive
 843          integer registers, then we can use that address directly.  */
 844       else if (n == 1
 845                || (n == 2 && !(SSE_CLASS_P (classes[0])
 846                                || SSE_CLASS_P (classes[1]))))
 847         {
 848           /* The argument is in a single register.  */
 849           if (SSE_CLASS_P (classes[0]))
 850             {
 851               avalue[i] = &reg_args->sse[ssecount];
 852               ssecount += n;
 853             }
 854           else
 855             {
 856               avalue[i] = &reg_args->gpr[gprcount];
 857               gprcount += n;
 858             }
 859         }
 860       /* Otherwise, allocate space to make them consecutive.  */
 861       else
 862         {
 863           char *a = alloca (16);
 864           unsigned int j;
 865
 866           avalue[i] = a;
 867           for (j = 0; j < n; j++, a += 8)
 868             {
 869               if (SSE_CLASS_P (classes[j]))
 870                 memcpy (a, &reg_args->sse[ssecount++], 8);
 871               else
 872                 memcpy (a, &reg_args->gpr[gprcount++], 8);
 873             }
 874         }
 875     }
 876
 877   /* Invoke the closure.  */
 878   fun (cif, rvalue, avalue, user_data);
 879
 880   /* Tell assembly how to perform return type promotions.  */
 881   return flags;
 882 }
 883
 884 #ifdef FFI_GO_CLOSURES
 885
 886 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
 887 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
 888
 889 #ifndef __ILP32__
 890 extern ffi_status
 891 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
 892                           void (*fun)(ffi_cif*, void*, void**, void*));
 893 #endif
 894
 895 ffi_status
 896 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
 897                      void (*fun)(ffi_cif*, void*, void**, void*))
 898 {
 899 #ifndef __ILP32__
 900   if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
 901     return ffi_prep_go_closure_efi64(closure, cif, fun);
 902 #endif
 903   if (cif->abi != FFI_UNIX64)
 904     return FFI_BAD_ABI;
 905
 906   closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
 907                     ? ffi_go_closure_unix64_sse
 908                     : ffi_go_closure_unix64);
 909   closure->cif = cif;
 910   closure->fun = fun;
 911
 912   return FFI_OK;
 913 }
 914
 915 #endif /* FFI_GO_CLOSURES */
 916
 917 #if defined(FFI_EXEC_STATIC_TRAMP)
 918 void *
 919 ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
 920 {
 921   extern void *trampoline_code_table;
 922
 923   *map_size = UNIX64_TRAMP_MAP_SIZE;
 924   *tramp_size = UNIX64_TRAMP_SIZE;
 925   return &trampoline_code_table;
 926 }
 927 #endif
 928
 929 #endif /* __x86_64__ */