gcc/config/aarch64/aarch64-simd-builtins.def

   1 /* Machine description for AArch64 architecture.
   2    Copyright (C) 2012-2019 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 /* In the list below, the BUILTIN_<ITERATOR> macros expand to create
  22    builtins for each of the modes described by <ITERATOR>.  When adding
  23    new builtins to this list, a helpful idiom to follow is to add
  24    a line for each pattern in the md file.  Thus, ADDP, which has one
  25    pattern defined for the VD_BHSI iterator, and one for DImode, has two
  26    entries below.
  27
  28    Parameter 1 is the 'type' of the intrinsic.  This is used to
  29    describe the type modifiers (for example; unsigned) applied to
  30    each of the parameters to the intrinsic function.
  31
  32    Parameter 2 is the name of the intrinsic.  This is appended
  33    to `__builtin_aarch64_<name><mode>` to give the intrinsic name
  34    as exported to the front-ends.
  35
  36    Parameter 3 describes how to map from the name to the CODE_FOR_
  37    macro holding the RTL pattern for the intrinsic.  This mapping is:
  38    0 - CODE_FOR_aarch64_<name><mode>
  39    1-9 - CODE_FOR_<name><mode><1-9>
  40    10 - CODE_FOR_<name><mode>.  */
  41
  42   BUILTIN_VDC (COMBINE, combine, 0)
  43   VAR1 (COMBINEP, combine, 0, di)
  44   BUILTIN_VB (BINOP, pmul, 0)
  45   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0)
  46   BUILTIN_VHSDF_DF (UNOP, sqrt, 2)
  47   BUILTIN_VD_BHSI (BINOP, addp, 0)
  48   VAR1 (UNOP, addp, 0, di)
  49   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2)
  50   BUILTIN_VDQ_BHSI (UNOP, clz, 2)
  51   BUILTIN_VS (UNOP, ctz, 2)
  52   BUILTIN_VB (UNOP, popcount, 2)
  53
  54   /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
  55   BUILTIN_VSDQ_I (BINOP, sqshl, 0)
  56   BUILTIN_VSDQ_I (BINOP_UUS, uqshl, 0)
  57   BUILTIN_VSDQ_I (BINOP, sqrshl, 0)
  58   BUILTIN_VSDQ_I (BINOP_UUS, uqrshl, 0)
  59   /* Implemented by aarch64_<su_optab><optab><mode>.  */
  60   BUILTIN_VSDQ_I (BINOP, sqadd, 0)
  61   BUILTIN_VSDQ_I (BINOPU, uqadd, 0)
  62   BUILTIN_VSDQ_I (BINOP, sqsub, 0)
  63   BUILTIN_VSDQ_I (BINOPU, uqsub, 0)
  64   /* Implemented by aarch64_<sur>qadd<mode>.  */
  65   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0)
  66   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0)
  67
  68   /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
  69   BUILTIN_VDC (GETREG, get_dregoi, 0)
  70   BUILTIN_VDC (GETREG, get_dregci, 0)
  71   BUILTIN_VDC (GETREG, get_dregxi, 0)
  72   VAR1 (GETREGP, get_dregoi, 0, di)
  73   VAR1 (GETREGP, get_dregci, 0, di)
  74   VAR1 (GETREGP, get_dregxi, 0, di)
  75   /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
  76   BUILTIN_VQ (GETREG, get_qregoi, 0)
  77   BUILTIN_VQ (GETREG, get_qregci, 0)
  78   BUILTIN_VQ (GETREG, get_qregxi, 0)
  79   VAR1 (GETREGP, get_qregoi, 0, v2di)
  80   VAR1 (GETREGP, get_qregci, 0, v2di)
  81   VAR1 (GETREGP, get_qregxi, 0, v2di)
  82   /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
  83   BUILTIN_VQ (SETREG, set_qregoi, 0)
  84   BUILTIN_VQ (SETREG, set_qregci, 0)
  85   BUILTIN_VQ (SETREG, set_qregxi, 0)
  86   VAR1 (SETREGP, set_qregoi, 0, v2di)
  87   VAR1 (SETREGP, set_qregci, 0, v2di)
  88   VAR1 (SETREGP, set_qregxi, 0, v2di)
  89   /* Implemented by aarch64_ld1x2<VQ:mode>. */
  90   BUILTIN_VQ (LOADSTRUCT, ld1x2, 0)
  91   /* Implemented by aarch64_ld1x2<VDC:mode>. */
  92   BUILTIN_VDC (LOADSTRUCT, ld1x2, 0)
  93   /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
  94   BUILTIN_VDC (LOADSTRUCT, ld2, 0)
  95   BUILTIN_VDC (LOADSTRUCT, ld3, 0)
  96   BUILTIN_VDC (LOADSTRUCT, ld4, 0)
  97   /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
  98   BUILTIN_VQ (LOADSTRUCT, ld2, 0)
  99   BUILTIN_VQ (LOADSTRUCT, ld3, 0)
 100   BUILTIN_VQ (LOADSTRUCT, ld4, 0)
 101   /* Implemented by aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>.  */
 102   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0)
 103   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0)
 104   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0)
 105   /* Implemented by aarch64_ld<VSTRUCT:nregs>_lane<VQ:mode>.  */
 106   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0)
 107   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0)
 108   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0)
 109   /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
 110   BUILTIN_VDC (STORESTRUCT, st2, 0)
 111   BUILTIN_VDC (STORESTRUCT, st3, 0)
 112   BUILTIN_VDC (STORESTRUCT, st4, 0)
 113   /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
 114   BUILTIN_VQ (STORESTRUCT, st2, 0)
 115   BUILTIN_VQ (STORESTRUCT, st3, 0)
 116   BUILTIN_VQ (STORESTRUCT, st4, 0)
 117
 118   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0)
 119   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0)
 120   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0)
 121
 122   BUILTIN_VQW (BINOP, saddl2, 0)
 123   BUILTIN_VQW (BINOP, uaddl2, 0)
 124   BUILTIN_VQW (BINOP, ssubl2, 0)
 125   BUILTIN_VQW (BINOP, usubl2, 0)
 126   BUILTIN_VQW (BINOP, saddw2, 0)
 127   BUILTIN_VQW (BINOP, uaddw2, 0)
 128   BUILTIN_VQW (BINOP, ssubw2, 0)
 129   BUILTIN_VQW (BINOP, usubw2, 0)
 130   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
 131   BUILTIN_VD_BHSI (BINOP, saddl, 0)
 132   BUILTIN_VD_BHSI (BINOP, uaddl, 0)
 133   BUILTIN_VD_BHSI (BINOP, ssubl, 0)
 134   BUILTIN_VD_BHSI (BINOP, usubl, 0)
 135   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
 136   BUILTIN_VD_BHSI (BINOP, saddw, 0)
 137   BUILTIN_VD_BHSI (BINOP, uaddw, 0)
 138   BUILTIN_VD_BHSI (BINOP, ssubw, 0)
 139   BUILTIN_VD_BHSI (BINOP, usubw, 0)
 140   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
 141   BUILTIN_VDQ_BHSI (BINOP, shadd, 0)
 142   BUILTIN_VDQ_BHSI (BINOP, shsub, 0)
 143   BUILTIN_VDQ_BHSI (BINOP, uhadd, 0)
 144   BUILTIN_VDQ_BHSI (BINOP, uhsub, 0)
 145   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0)
 146   BUILTIN_VDQ_BHSI (BINOP, urhadd, 0)
 147   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
 148   BUILTIN_VQN (BINOP, addhn, 0)
 149   BUILTIN_VQN (BINOP, subhn, 0)
 150   BUILTIN_VQN (BINOP, raddhn, 0)
 151   BUILTIN_VQN (BINOP, rsubhn, 0)
 152   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
 153   BUILTIN_VQN (TERNOP, addhn2, 0)
 154   BUILTIN_VQN (TERNOP, subhn2, 0)
 155   BUILTIN_VQN (TERNOP, raddhn2, 0)
 156   BUILTIN_VQN (TERNOP, rsubhn2, 0)
 157
 158   BUILTIN_VSQN_HSDI (UNOP, sqmovun, 0)
 159   /* Implemented by aarch64_<sur>qmovn<mode>.  */
 160   BUILTIN_VSQN_HSDI (UNOP, sqmovn, 0)
 161   BUILTIN_VSQN_HSDI (UNOP, uqmovn, 0)
 162   /* Implemented by aarch64_s<optab><mode>.  */
 163   BUILTIN_VSDQ_I (UNOP, sqabs, 0)
 164   BUILTIN_VSDQ_I (UNOP, sqneg, 0)
 165
 166   /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
 167   BUILTIN_VSD_HSI (TERNOP, sqdmlal, 0)
 168   BUILTIN_VSD_HSI (TERNOP, sqdmlsl, 0)
 169   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_lane<mode>.  */
 170   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_lane, 0)
 171   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_lane, 0)
 172   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_laneq<mode>.  */
 173   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_laneq, 0)
 174   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_laneq, 0)
 175   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
 176   BUILTIN_VD_HSI (TERNOP, sqdmlal_n, 0)
 177   BUILTIN_VD_HSI (TERNOP, sqdmlsl_n, 0)
 178
 179   BUILTIN_VQ_HSI (TERNOP, sqdmlal2, 0)
 180   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2, 0)
 181   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_lane, 0)
 182   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_lane, 0)
 183   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_laneq, 0)
 184   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_laneq, 0)
 185   BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n, 0)
 186   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n, 0)
 187
 188   BUILTIN_VSD_HSI (BINOP, sqdmull, 0)
 189   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_lane, 0)
 190   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_laneq, 0)
 191   BUILTIN_VD_HSI (BINOP, sqdmull_n, 0)
 192   BUILTIN_VQ_HSI (BINOP, sqdmull2, 0)
 193   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_lane, 0)
 194   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_laneq, 0)
 195   BUILTIN_VQ_HSI (BINOP, sqdmull2_n, 0)
 196   /* Implemented by aarch64_sq<r>dmulh<mode>.  */
 197   BUILTIN_VSDQ_HSI (BINOP, sqdmulh, 0)
 198   BUILTIN_VSDQ_HSI (BINOP, sqrdmulh, 0)
 199   /* Implemented by aarch64_sq<r>dmulh_lane<q><mode>.  */
 200   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_lane, 0)
 201   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_laneq, 0)
 202   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_lane, 0)
 203   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_laneq, 0)
 204
 205   BUILTIN_VSDQ_I_DI (BINOP, ashl, 3)
 206   /* Implemented by aarch64_<sur>shl<mode>.  */
 207   BUILTIN_VSDQ_I_DI (BINOP, sshl, 0)
 208   BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0)
 209   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0)
 210   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0)
 211
 212   /* Implemented by aarch64_<sur><dotprod>{_lane}{q}<dot_mode>.  */
 213   BUILTIN_VB (TERNOP, sdot, 0)
 214   BUILTIN_VB (TERNOPU, udot, 0)
 215   BUILTIN_VB (QUADOP_LANE, sdot_lane, 0)
 216   BUILTIN_VB (QUADOPU_LANE, udot_lane, 0)
 217   BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0)
 218   BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0)
 219
 220   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3)
 221   VAR1 (SHIFTIMM, ashr_simd, 0, di)
 222   BUILTIN_VDQ_I (SHIFTIMM, lshr, 3)
 223   VAR1 (USHIFTIMM, lshr_simd, 0, di)
 224   /* Implemented by aarch64_<sur>shr_n<mode>.  */
 225   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0)
 226   BUILTIN_VSDQ_I_DI (USHIFTIMM, urshr_n, 0)
 227   /* Implemented by aarch64_<sur>sra_n<mode>.  */
 228   BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n, 0)
 229   BUILTIN_VSDQ_I_DI (USHIFTACC, usra_n, 0)
 230   BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n, 0)
 231   BUILTIN_VSDQ_I_DI (USHIFTACC, ursra_n, 0)
 232   /* Implemented by aarch64_<sur>shll_n<mode>.  */
 233   BUILTIN_VD_BHSI (SHIFTIMM, sshll_n, 0)
 234   BUILTIN_VD_BHSI (USHIFTIMM, ushll_n, 0)
 235   /* Implemented by aarch64_<sur>shll2_n<mode>.  */
 236   BUILTIN_VQW (SHIFTIMM, sshll2_n, 0)
 237   BUILTIN_VQW (SHIFTIMM, ushll2_n, 0)
 238   /* Implemented by aarch64_<sur>q<r>shr<u>n_n<mode>.  */
 239   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrun_n, 0)
 240   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrun_n, 0)
 241   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrn_n, 0)
 242   BUILTIN_VSQN_HSDI (USHIFTIMM, uqshrn_n, 0)
 243   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrn_n, 0)
 244   BUILTIN_VSQN_HSDI (USHIFTIMM, uqrshrn_n, 0)
 245   /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
 246   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n, 0)
 247   BUILTIN_VSDQ_I_DI (USHIFTACC, usri_n, 0)
 248   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n, 0)
 249   VAR2 (SHIFTINSERTP, ssli_n, 0, di, v2di)
 250   BUILTIN_VSDQ_I_DI (USHIFTACC, usli_n, 0)
 251   /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
 252   BUILTIN_VSDQ_I (SHIFTIMM_USS, sqshlu_n, 0)
 253   BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n, 0)
 254   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0)
 255
 256   /* Implemented by aarch64_reduc_plus_<mode>.  */
 257   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10)
 258
 259   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
 260   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10)
 261   BUILTIN_VDQIF_F16 (UNOP, reduc_smin_scal_, 10)
 262   BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10)
 263   BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10)
 264   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10)
 265   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10)
 266
 267   /* Implemented by <maxmin_uns><mode>3.
 268      smax variants map to fmaxnm,
 269      smax_nan variants map to fmax.  */
 270   BUILTIN_VDQ_BHSI (BINOP, smax, 3)
 271   BUILTIN_VDQ_BHSI (BINOP, smin, 3)
 272   BUILTIN_VDQ_BHSI (BINOP, umax, 3)
 273   BUILTIN_VDQ_BHSI (BINOP, umin, 3)
 274   BUILTIN_VHSDF_DF (BINOP, smax_nan, 3)
 275   BUILTIN_VHSDF_DF (BINOP, smin_nan, 3)
 276
 277   /* Implemented by <maxmin_uns><mode>3.  */
 278   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3)
 279   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3)
 280
 281   /* Implemented by aarch64_<maxmin_uns>p<mode>.  */
 282   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0)
 283   BUILTIN_VDQ_BHSI (BINOP, sminp, 0)
 284   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0)
 285   BUILTIN_VDQ_BHSI (BINOP, uminp, 0)
 286   BUILTIN_VHSDF (BINOP, smaxp, 0)
 287   BUILTIN_VHSDF (BINOP, sminp, 0)
 288   BUILTIN_VHSDF (BINOP, smax_nanp, 0)
 289   BUILTIN_VHSDF (BINOP, smin_nanp, 0)
 290
 291   /* Implemented by <frint_pattern><mode>2.  */
 292   BUILTIN_VHSDF (UNOP, btrunc, 2)
 293   BUILTIN_VHSDF (UNOP, ceil, 2)
 294   BUILTIN_VHSDF (UNOP, floor, 2)
 295   BUILTIN_VHSDF (UNOP, nearbyint, 2)
 296   BUILTIN_VHSDF (UNOP, rint, 2)
 297   BUILTIN_VHSDF (UNOP, round, 2)
 298   BUILTIN_VHSDF_DF (UNOP, frintn, 2)
 299
 300   VAR1 (UNOP, btrunc, 2, hf)
 301   VAR1 (UNOP, ceil, 2, hf)
 302   VAR1 (UNOP, floor, 2, hf)
 303   VAR1 (UNOP, frintn, 2, hf)
 304   VAR1 (UNOP, nearbyint, 2, hf)
 305   VAR1 (UNOP, rint, 2, hf)
 306   VAR1 (UNOP, round, 2, hf)
 307
 308   /* Implemented by l<fcvt_pattern><su_optab><VQDF:mode><vcvt_target>2.  */
 309   VAR1 (UNOP, lbtruncv4hf, 2, v4hi)
 310   VAR1 (UNOP, lbtruncv8hf, 2, v8hi)
 311   VAR1 (UNOP, lbtruncv2sf, 2, v2si)
 312   VAR1 (UNOP, lbtruncv4sf, 2, v4si)
 313   VAR1 (UNOP, lbtruncv2df, 2, v2di)
 314
 315   VAR1 (UNOPUS, lbtruncuv4hf, 2, v4hi)
 316   VAR1 (UNOPUS, lbtruncuv8hf, 2, v8hi)
 317   VAR1 (UNOPUS, lbtruncuv2sf, 2, v2si)
 318   VAR1 (UNOPUS, lbtruncuv4sf, 2, v4si)
 319   VAR1 (UNOPUS, lbtruncuv2df, 2, v2di)
 320
 321   VAR1 (UNOP, lroundv4hf, 2, v4hi)
 322   VAR1 (UNOP, lroundv8hf, 2, v8hi)
 323   VAR1 (UNOP, lroundv2sf, 2, v2si)
 324   VAR1 (UNOP, lroundv4sf, 2, v4si)
 325   VAR1 (UNOP, lroundv2df, 2, v2di)
 326   /* Implemented by l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2.  */
 327   BUILTIN_GPI_I16 (UNOP, lroundhf, 2)
 328   VAR1 (UNOP, lroundsf, 2, si)
 329   VAR1 (UNOP, lrounddf, 2, di)
 330
 331   VAR1 (UNOPUS, lrounduv4hf, 2, v4hi)
 332   VAR1 (UNOPUS, lrounduv8hf, 2, v8hi)
 333   VAR1 (UNOPUS, lrounduv2sf, 2, v2si)
 334   VAR1 (UNOPUS, lrounduv4sf, 2, v4si)
 335   VAR1 (UNOPUS, lrounduv2df, 2, v2di)
 336   BUILTIN_GPI_I16 (UNOPUS, lrounduhf, 2)
 337   VAR1 (UNOPUS, lroundusf, 2, si)
 338   VAR1 (UNOPUS, lroundudf, 2, di)
 339
 340   VAR1 (UNOP, lceilv4hf, 2, v4hi)
 341   VAR1 (UNOP, lceilv8hf, 2, v8hi)
 342   VAR1 (UNOP, lceilv2sf, 2, v2si)
 343   VAR1 (UNOP, lceilv4sf, 2, v4si)
 344   VAR1 (UNOP, lceilv2df, 2, v2di)
 345   BUILTIN_GPI_I16 (UNOP, lceilhf, 2)
 346
 347   VAR1 (UNOPUS, lceiluv4hf, 2, v4hi)
 348   VAR1 (UNOPUS, lceiluv8hf, 2, v8hi)
 349   VAR1 (UNOPUS, lceiluv2sf, 2, v2si)
 350   VAR1 (UNOPUS, lceiluv4sf, 2, v4si)
 351   VAR1 (UNOPUS, lceiluv2df, 2, v2di)
 352   BUILTIN_GPI_I16 (UNOPUS, lceiluhf, 2)
 353   VAR1 (UNOPUS, lceilusf, 2, si)
 354   VAR1 (UNOPUS, lceiludf, 2, di)
 355
 356   VAR1 (UNOP, lfloorv4hf, 2, v4hi)
 357   VAR1 (UNOP, lfloorv8hf, 2, v8hi)
 358   VAR1 (UNOP, lfloorv2sf, 2, v2si)
 359   VAR1 (UNOP, lfloorv4sf, 2, v4si)
 360   VAR1 (UNOP, lfloorv2df, 2, v2di)
 361   BUILTIN_GPI_I16 (UNOP, lfloorhf, 2)
 362
 363   VAR1 (UNOPUS, lflooruv4hf, 2, v4hi)
 364   VAR1 (UNOPUS, lflooruv8hf, 2, v8hi)
 365   VAR1 (UNOPUS, lflooruv2sf, 2, v2si)
 366   VAR1 (UNOPUS, lflooruv4sf, 2, v4si)
 367   VAR1 (UNOPUS, lflooruv2df, 2, v2di)
 368   BUILTIN_GPI_I16 (UNOPUS, lflooruhf, 2)
 369   VAR1 (UNOPUS, lfloorusf, 2, si)
 370   VAR1 (UNOPUS, lfloorudf, 2, di)
 371
 372   VAR1 (UNOP, lfrintnv4hf, 2, v4hi)
 373   VAR1 (UNOP, lfrintnv8hf, 2, v8hi)
 374   VAR1 (UNOP, lfrintnv2sf, 2, v2si)
 375   VAR1 (UNOP, lfrintnv4sf, 2, v4si)
 376   VAR1 (UNOP, lfrintnv2df, 2, v2di)
 377   BUILTIN_GPI_I16 (UNOP, lfrintnhf, 2)
 378   VAR1 (UNOP, lfrintnsf, 2, si)
 379   VAR1 (UNOP, lfrintndf, 2, di)
 380
 381   VAR1 (UNOPUS, lfrintnuv4hf, 2, v4hi)
 382   VAR1 (UNOPUS, lfrintnuv8hf, 2, v8hi)
 383   VAR1 (UNOPUS, lfrintnuv2sf, 2, v2si)
 384   VAR1 (UNOPUS, lfrintnuv4sf, 2, v4si)
 385   VAR1 (UNOPUS, lfrintnuv2df, 2, v2di)
 386   BUILTIN_GPI_I16 (UNOPUS, lfrintnuhf, 2)
 387   VAR1 (UNOPUS, lfrintnusf, 2, si)
 388   VAR1 (UNOPUS, lfrintnudf, 2, di)
 389
 390   /* Implemented by <optab><fcvt_target><VDQF:mode>2.  */
 391   VAR1 (UNOP, floatv4hi, 2, v4hf)
 392   VAR1 (UNOP, floatv8hi, 2, v8hf)
 393   VAR1 (UNOP, floatv2si, 2, v2sf)
 394   VAR1 (UNOP, floatv4si, 2, v4sf)
 395   VAR1 (UNOP, floatv2di, 2, v2df)
 396
 397   VAR1 (UNOP, floatunsv4hi, 2, v4hf)
 398   VAR1 (UNOP, floatunsv8hi, 2, v8hf)
 399   VAR1 (UNOP, floatunsv2si, 2, v2sf)
 400   VAR1 (UNOP, floatunsv4si, 2, v4sf)
 401   VAR1 (UNOP, floatunsv2di, 2, v2df)
 402
 403   VAR5 (UNOPU, bswap, 2, v4hi, v8hi, v2si, v4si, v2di)
 404
 405   BUILTIN_VB (UNOP, rbit, 0)
 406
 407   /* Implemented by
 408      aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>.  */
 409   BUILTIN_VALL (BINOP, zip1, 0)
 410   BUILTIN_VALL (BINOP, zip2, 0)
 411   BUILTIN_VALL (BINOP, uzp1, 0)
 412   BUILTIN_VALL (BINOP, uzp2, 0)
 413   BUILTIN_VALL (BINOP, trn1, 0)
 414   BUILTIN_VALL (BINOP, trn2, 0)
 415
 416   BUILTIN_GPF_F16 (UNOP, frecpe, 0)
 417   BUILTIN_GPF_F16 (UNOP, frecpx, 0)
 418
 419   BUILTIN_VDQ_SI (UNOP, urecpe, 0)
 420
 421   BUILTIN_VHSDF (UNOP, frecpe, 0)
 422   BUILTIN_VHSDF_HSDF (BINOP, frecps, 0)
 423
 424   /* Implemented by a mixture of abs2 patterns.  Note the DImode builtin is
 425      only ever used for the int64x1_t intrinsic, there is no scalar version.  */
 426   BUILTIN_VSDQ_I_DI (UNOP, abs, 0)
 427   BUILTIN_VHSDF (UNOP, abs, 2)
 428   VAR1 (UNOP, abs, 2, hf)
 429
 430   BUILTIN_VQ_HSF (UNOP, vec_unpacks_hi_, 10)
 431   VAR1 (BINOP, float_truncate_hi_, 0, v4sf)
 432   VAR1 (BINOP, float_truncate_hi_, 0, v8hf)
 433
 434   VAR1 (UNOP, float_extend_lo_, 0, v2df)
 435   VAR1 (UNOP, float_extend_lo_,  0, v4sf)
 436   BUILTIN_VDF (UNOP, float_truncate_lo_, 0)
 437
 438   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
 439   BUILTIN_VALL_F16 (LOAD1, ld1, 0)
 440   VAR1(STORE1P, ld1, 0, v2di)
 441
 442   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
 443   BUILTIN_VALL_F16 (STORE1, st1, 0)
 444   VAR1(STORE1P, st1, 0, v2di)
 445
 446   /* Implemented by aarch64_ld1x3<VALLDIF:mode>.  */
 447   BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0)
 448
 449   /* Implemented by aarch64_st1x2<VALLDIF:mode>.  */
 450   BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0)
 451
 452   /* Implemented by aarch64_st1x3<VALLDIF:mode>.  */
 453   BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0)
 454
 455   /* Implemented by fma<mode>4.  */
 456   BUILTIN_VHSDF (TERNOP, fma, 4)
 457   VAR1 (TERNOP, fma, 4, hf)
 458   /* Implemented by fnma<mode>4.  */
 459   BUILTIN_VHSDF (TERNOP, fnma, 4)
 460   VAR1 (TERNOP, fnma, 4, hf)
 461
 462   /* Implemented by aarch64_simd_bsl<mode>.  */
 463   BUILTIN_VDQQH (BSL_P, simd_bsl, 0)
 464   VAR2 (BSL_P, simd_bsl,0, di, v2di)
 465   BUILTIN_VSDQ_I_DI (BSL_U, simd_bsl, 0)
 466   BUILTIN_VALLDIF (BSL_S, simd_bsl, 0)
 467
 468   /* Implemented by aarch64_crypto_aes<op><mode>.  */
 469   VAR1 (BINOPU, crypto_aese, 0, v16qi)
 470   VAR1 (BINOPU, crypto_aesd, 0, v16qi)
 471   VAR1 (UNOPU, crypto_aesmc, 0, v16qi)
 472   VAR1 (UNOPU, crypto_aesimc, 0, v16qi)
 473
 474   /* Implemented by aarch64_crypto_sha1<op><mode>.  */
 475   VAR1 (UNOPU, crypto_sha1h, 0, si)
 476   VAR1 (BINOPU, crypto_sha1su1, 0, v4si)
 477   VAR1 (TERNOPU, crypto_sha1c, 0, v4si)
 478   VAR1 (TERNOPU, crypto_sha1m, 0, v4si)
 479   VAR1 (TERNOPU, crypto_sha1p, 0, v4si)
 480   VAR1 (TERNOPU, crypto_sha1su0, 0, v4si)
 481
 482   /* Implemented by aarch64_crypto_sha256<op><mode>.  */
 483   VAR1 (TERNOPU, crypto_sha256h, 0, v4si)
 484   VAR1 (TERNOPU, crypto_sha256h2, 0, v4si)
 485   VAR1 (BINOPU, crypto_sha256su0, 0, v4si)
 486   VAR1 (TERNOPU, crypto_sha256su1, 0, v4si)
 487
 488   /* Implemented by aarch64_crypto_pmull<mode>.  */
 489   VAR1 (BINOPP, crypto_pmull, 0, di)
 490   VAR1 (BINOPP, crypto_pmull, 0, v2di)
 491
 492   /* Implemented by aarch64_tbl3<mode>.  */
 493   VAR1 (BINOP, tbl3, 0, v8qi)
 494   VAR1 (BINOP, tbl3, 0, v16qi)
 495
 496   /* Implemented by aarch64_qtbl3<mode>.  */
 497   VAR1 (BINOP, qtbl3, 0, v8qi)
 498   VAR1 (BINOP, qtbl3, 0, v16qi)
 499
 500   /* Implemented by aarch64_qtbl4<mode>.  */
 501   VAR1 (BINOP, qtbl4, 0, v8qi)
 502   VAR1 (BINOP, qtbl4, 0, v16qi)
 503
 504   /* Implemented by aarch64_tbx4<mode>.  */
 505   VAR1 (TERNOP, tbx4, 0, v8qi)
 506   VAR1 (TERNOP, tbx4, 0, v16qi)
 507
 508   /* Implemented by aarch64_qtbx3<mode>.  */
 509   VAR1 (TERNOP, qtbx3, 0, v8qi)
 510   VAR1 (TERNOP, qtbx3, 0, v16qi)
 511
 512   /* Implemented by aarch64_qtbx4<mode>.  */
 513   VAR1 (TERNOP, qtbx4, 0, v8qi)
 514   VAR1 (TERNOP, qtbx4, 0, v16qi)
 515
 516   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 517
 518   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>.  */
 519   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlah, 0)
 520   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlsh, 0)
 521
 522   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>.  */
 523   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_lane, 0)
 524   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_lane, 0)
 525
 526   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>.  */
 527   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_laneq, 0)
 528   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_laneq, 0)
 529
 530   /* Implemented by <FCVT_F2FIXED/FIXED2F:fcvt_fixed_insn><*><*>3.  */
 531   BUILTIN_VSDQ_HSDI (SHIFTIMM, scvtf, 3)
 532   BUILTIN_VSDQ_HSDI (FCVTIMM_SUS, ucvtf, 3)
 533   BUILTIN_VHSDF_HSDF (SHIFTIMM, fcvtzs, 3)
 534   BUILTIN_VHSDF_HSDF (SHIFTIMM_USS, fcvtzu, 3)
 535   VAR1 (SHIFTIMM, scvtfsi, 3, hf)
 536   VAR1 (SHIFTIMM, scvtfdi, 3, hf)
 537   VAR1 (FCVTIMM_SUS, ucvtfsi, 3, hf)
 538   VAR1 (FCVTIMM_SUS, ucvtfdi, 3, hf)
 539   BUILTIN_GPI (SHIFTIMM, fcvtzshf, 3)
 540   BUILTIN_GPI (SHIFTIMM_USS, fcvtzuhf, 3)
 541
 542   /* Implemented by aarch64_rsqrte<mode>.  */
 543   BUILTIN_VHSDF_HSDF (UNOP, rsqrte, 0)
 544
 545   /* Implemented by aarch64_rsqrts<mode>.  */
 546   BUILTIN_VHSDF_HSDF (BINOP, rsqrts, 0)
 547
 548   /* Implemented by fabd<mode>3.  */
 549   BUILTIN_VHSDF_HSDF (BINOP, fabd, 3)
 550
 551   /* Implemented by aarch64_faddp<mode>.  */
 552   BUILTIN_VHSDF (BINOP, faddp, 0)
 553
 554   /* Implemented by aarch64_cm<optab><mode>.  */
 555   BUILTIN_VHSDF_HSDF (BINOP_USS, cmeq, 0)
 556   BUILTIN_VHSDF_HSDF (BINOP_USS, cmge, 0)
 557   BUILTIN_VHSDF_HSDF (BINOP_USS, cmgt, 0)
 558   BUILTIN_VHSDF_HSDF (BINOP_USS, cmle, 0)
 559   BUILTIN_VHSDF_HSDF (BINOP_USS, cmlt, 0)
 560
 561   /* Implemented by neg<mode>2.  */
 562   BUILTIN_VHSDF_HSDF (UNOP, neg, 2)
 563
 564   /* Implemented by aarch64_fac<optab><mode>.  */
 565   BUILTIN_VHSDF_HSDF (BINOP_USS, faclt, 0)
 566   BUILTIN_VHSDF_HSDF (BINOP_USS, facle, 0)
 567   BUILTIN_VHSDF_HSDF (BINOP_USS, facgt, 0)
 568   BUILTIN_VHSDF_HSDF (BINOP_USS, facge, 0)
 569
 570   /* Implemented by sqrt<mode>2.  */
 571   VAR1 (UNOP, sqrt, 2, hf)
 572
 573   /* Implemented by <optab><mode>hf2.  */
 574   VAR1 (UNOP, floatdi, 2, hf)
 575   VAR1 (UNOP, floatsi, 2, hf)
 576   VAR1 (UNOP, floathi, 2, hf)
 577   VAR1 (UNOPUS, floatunsdi, 2, hf)
 578   VAR1 (UNOPUS, floatunssi, 2, hf)
 579   VAR1 (UNOPUS, floatunshi, 2, hf)
 580   BUILTIN_GPI_I16 (UNOP, fix_trunchf, 2)
 581   BUILTIN_GPI (UNOP, fix_truncsf, 2)
 582   BUILTIN_GPI (UNOP, fix_truncdf, 2)
 583   BUILTIN_GPI_I16 (UNOPUS, fixuns_trunchf, 2)
 584   BUILTIN_GPI (UNOPUS, fixuns_truncsf, 2)
 585   BUILTIN_GPI (UNOPUS, fixuns_truncdf, 2)
 586
 587   /* Implemented by aarch64_sm3ss1qv4si.  */
 588   VAR1 (TERNOPU, sm3ss1q, 0, v4si)
 589   /* Implemented by aarch64_sm3tt<sm3tt_op>qv4si.  */
 590   VAR1 (QUADOPUI, sm3tt1aq, 0, v4si)
 591   VAR1 (QUADOPUI, sm3tt1bq, 0, v4si)
 592   VAR1 (QUADOPUI, sm3tt2aq, 0, v4si)
 593   VAR1 (QUADOPUI, sm3tt2bq, 0, v4si)
 594   /* Implemented by aarch64_sm3partw<sm3part_op>qv4si.  */
 595   VAR1 (TERNOPU, sm3partw1q, 0, v4si)
 596   VAR1 (TERNOPU, sm3partw2q, 0, v4si)
 597   /* Implemented by aarch64_sm4eqv4si.  */
 598   VAR1 (BINOPU, sm4eq, 0, v4si)
 599   /* Implemented by aarch64_sm4ekeyqv4si.  */
 600   VAR1 (BINOPU, sm4ekeyq, 0, v4si)
 601   /* Implemented by aarch64_crypto_sha512hqv2di.  */
 602   VAR1 (TERNOPU, crypto_sha512hq, 0, v2di)
 603   /* Implemented by aarch64_sha512h2qv2di.  */
 604   VAR1 (TERNOPU, crypto_sha512h2q, 0, v2di)
 605   /* Implemented by aarch64_crypto_sha512su0qv2di.  */
 606   VAR1 (BINOPU, crypto_sha512su0q, 0, v2di)
 607   /* Implemented by aarch64_crypto_sha512su1qv2di.  */
 608   VAR1 (TERNOPU, crypto_sha512su1q, 0, v2di)
 609   /* Implemented by eor3q<mode>4.  */
 610   BUILTIN_VQ_I (TERNOPU, eor3q, 4)
 611   BUILTIN_VQ_I (TERNOP, eor3q, 4)
 612   /* Implemented by aarch64_rax1qv2di.  */
 613   VAR1 (BINOPU, rax1q, 0, v2di)
 614   /* Implemented by aarch64_xarqv2di.  */
 615   VAR1 (TERNOPUI, xarq, 0, v2di)
 616   /* Implemented by bcaxq<mode>4.  */
 617   BUILTIN_VQ_I (TERNOPU, bcaxq, 4)
 618   BUILTIN_VQ_I (TERNOP, bcaxq, 4)
 619
 620   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_low<mode>.  */
 621   VAR1 (TERNOP, fmlal_low, 0, v2sf)
 622   VAR1 (TERNOP, fmlsl_low, 0, v2sf)
 623   VAR1 (TERNOP, fmlalq_low, 0, v4sf)
 624   VAR1 (TERNOP, fmlslq_low, 0, v4sf)
 625   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_high<mode>.  */
 626   VAR1 (TERNOP, fmlal_high, 0, v2sf)
 627   VAR1 (TERNOP, fmlsl_high, 0, v2sf)
 628   VAR1 (TERNOP, fmlalq_high, 0, v4sf)
 629   VAR1 (TERNOP, fmlslq_high, 0, v4sf)
 630   /* Implemented by aarch64_fml<f16mac1>l_lane_lowv2sf.  */
 631   VAR1 (QUADOP_LANE, fmlal_lane_low, 0, v2sf)
 632   VAR1 (QUADOP_LANE, fmlsl_lane_low, 0, v2sf)
 633   /* Implemented by aarch64_fml<f16mac1>l_laneq_lowv2sf.  */
 634   VAR1 (QUADOP_LANE, fmlal_laneq_low, 0, v2sf)
 635   VAR1 (QUADOP_LANE, fmlsl_laneq_low, 0, v2sf)
 636   /* Implemented by aarch64_fml<f16mac1>lq_lane_lowv4sf.  */
 637   VAR1 (QUADOP_LANE, fmlalq_lane_low, 0, v4sf)
 638   VAR1 (QUADOP_LANE, fmlslq_lane_low, 0, v4sf)
 639   /* Implemented by aarch64_fml<f16mac1>lq_laneq_lowv4sf.  */
 640   VAR1 (QUADOP_LANE, fmlalq_laneq_low, 0, v4sf)
 641   VAR1 (QUADOP_LANE, fmlslq_laneq_low, 0, v4sf)
 642   /* Implemented by aarch64_fml<f16mac1>l_lane_highv2sf.  */
 643   VAR1 (QUADOP_LANE, fmlal_lane_high, 0, v2sf)
 644   VAR1 (QUADOP_LANE, fmlsl_lane_high, 0, v2sf)
 645   /* Implemented by aarch64_fml<f16mac1>l_laneq_highv2sf.  */
 646   VAR1 (QUADOP_LANE, fmlal_laneq_high, 0, v2sf)
 647   VAR1 (QUADOP_LANE, fmlsl_laneq_high, 0, v2sf)
 648   /* Implemented by aarch64_fml<f16mac1>lq_lane_highv4sf.  */
 649   VAR1 (QUADOP_LANE, fmlalq_lane_high, 0, v4sf)
 650   VAR1 (QUADOP_LANE, fmlslq_lane_high, 0, v4sf)
 651   /* Implemented by aarch64_fml<f16mac1>lq_laneq_highv4sf.  */
 652   VAR1 (QUADOP_LANE, fmlalq_laneq_high, 0, v4sf)
 653   VAR1 (QUADOP_LANE, fmlslq_laneq_high, 0, v4sf)