gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
   3 ;;   Free Software Foundation, Inc.
   4 ;; Written by CodeSourcery.
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published by
  10 ;; the Free Software Foundation; either version 3, or (at your option)
  11 ;; any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but
  14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 ;; General Public License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22 ;; Enumerators for unspecs.
  23 (define_c_enum "unspec" [
  24   UNSPEC_ASHIFT_SIGNED
  25   UNSPEC_ASHIFT_UNSIGNED
  26   UNSPEC_VABD
  27   UNSPEC_VABDL
  28   UNSPEC_VADD
  29   UNSPEC_VADDHN
  30   UNSPEC_VADDL
  31   UNSPEC_VADDW
  32   UNSPEC_VBSL
  33   UNSPEC_VCAGE
  34   UNSPEC_VCAGT
  35   UNSPEC_VCEQ
  36   UNSPEC_VCGE
  37   UNSPEC_VCGEU
  38   UNSPEC_VCGT
  39   UNSPEC_VCGTU
  40   UNSPEC_VCLS
  41   UNSPEC_VCONCAT
  42   UNSPEC_VCVT
  43   UNSPEC_VCVT_N
  44   UNSPEC_VEXT
  45   UNSPEC_VHADD
  46   UNSPEC_VHSUB
  47   UNSPEC_VLD1
  48   UNSPEC_VLD1_LANE
  49   UNSPEC_VLD2
  50   UNSPEC_VLD2_DUP
  51   UNSPEC_VLD2_LANE
  52   UNSPEC_VLD3
  53   UNSPEC_VLD3A
  54   UNSPEC_VLD3B
  55   UNSPEC_VLD3_DUP
  56   UNSPEC_VLD3_LANE
  57   UNSPEC_VLD4
  58   UNSPEC_VLD4A
  59   UNSPEC_VLD4B
  60   UNSPEC_VLD4_DUP
  61   UNSPEC_VLD4_LANE
  62   UNSPEC_VMAX
  63   UNSPEC_VMIN
  64   UNSPEC_VMLA
  65   UNSPEC_VMLAL
  66   UNSPEC_VMLA_LANE
  67   UNSPEC_VMLAL_LANE
  68   UNSPEC_VMLS
  69   UNSPEC_VMLSL
  70   UNSPEC_VMLS_LANE
  71   UNSPEC_VMLSL_LANE
  72   UNSPEC_VMOVL
  73   UNSPEC_VMOVN
  74   UNSPEC_VMUL
  75   UNSPEC_VMULL
  76   UNSPEC_VMUL_LANE
  77   UNSPEC_VMULL_LANE
  78   UNSPEC_VPADAL
  79   UNSPEC_VPADD
  80   UNSPEC_VPADDL
  81   UNSPEC_VPMAX
  82   UNSPEC_VPMIN
  83   UNSPEC_VPSMAX
  84   UNSPEC_VPSMIN
  85   UNSPEC_VPUMAX
  86   UNSPEC_VPUMIN
  87   UNSPEC_VQABS
  88   UNSPEC_VQADD
  89   UNSPEC_VQDMLAL
  90   UNSPEC_VQDMLAL_LANE
  91   UNSPEC_VQDMLSL
  92   UNSPEC_VQDMLSL_LANE
  93   UNSPEC_VQDMULH
  94   UNSPEC_VQDMULH_LANE
  95   UNSPEC_VQDMULL
  96   UNSPEC_VQDMULL_LANE
  97   UNSPEC_VQMOVN
  98   UNSPEC_VQMOVUN
  99   UNSPEC_VQNEG
 100   UNSPEC_VQSHL
 101   UNSPEC_VQSHL_N
 102   UNSPEC_VQSHLU_N
 103   UNSPEC_VQSHRN_N
 104   UNSPEC_VQSHRUN_N
 105   UNSPEC_VQSUB
 106   UNSPEC_VRECPE
 107   UNSPEC_VRECPS
 108   UNSPEC_VREV16
 109   UNSPEC_VREV32
 110   UNSPEC_VREV64
 111   UNSPEC_VRSQRTE
 112   UNSPEC_VRSQRTS
 113   UNSPEC_VSHL
 114   UNSPEC_VSHLL_N
 115   UNSPEC_VSHL_N
 116   UNSPEC_VSHR_N
 117   UNSPEC_VSHRN_N
 118   UNSPEC_VSLI
 119   UNSPEC_VSRA_N
 120   UNSPEC_VSRI
 121   UNSPEC_VST1
 122   UNSPEC_VST1_LANE
 123   UNSPEC_VST2
 124   UNSPEC_VST2_LANE
 125   UNSPEC_VST3
 126   UNSPEC_VST3A
 127   UNSPEC_VST3B
 128   UNSPEC_VST3_LANE
 129   UNSPEC_VST4
 130   UNSPEC_VST4A
 131   UNSPEC_VST4B
 132   UNSPEC_VST4_LANE
 133   UNSPEC_VSTRUCTDUMMY
 134   UNSPEC_VSUB
 135   UNSPEC_VSUBHN
 136   UNSPEC_VSUBL
 137   UNSPEC_VSUBW
 138   UNSPEC_VTBL
 139   UNSPEC_VTBX
 140   UNSPEC_VTRN1
 141   UNSPEC_VTRN2
 142   UNSPEC_VTST
 143   UNSPEC_VUZP1
 144   UNSPEC_VUZP2
 145   UNSPEC_VZIP1
 146   UNSPEC_VZIP2
 147   UNSPEC_MISALIGNED_ACCESS
 148   UNSPEC_VCLE
 149   UNSPEC_VCLT
 150 ])
 151
 152
 153 ;; Attribute used to permit string comparisons against <VQH_mnem> in
 154 ;; neon_type attribute definitions.
 155 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
 156
 157 (define_insn "*neon_mov<mode>"
 158   [(set (match_operand:VDX 0 "nonimmediate_operand"
 159           "=w,Uv,w, w,  ?r,?w,?r,?r, ?Us")
 160         (match_operand:VDX 1 "general_operand"
 161           " w,w, Dn,Uvi, w, r, r, Usi,r"))]
 162   "TARGET_NEON
 163    && (register_operand (operands[0], <MODE>mode)
 164        || register_operand (operands[1], <MODE>mode))"
 165 {
 166   if (which_alternative == 2)
 167     {
 168       int width, is_valid;
 169       static char templ[40];
 170
 171       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 172         &operands[1], &width);
 173
 174       gcc_assert (is_valid != 0);
 175
 176       if (width == 0)
 177         return "vmov.f32\t%P0, %1  @ <mode>";
 178       else
 179         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
 180
 181       return templ;
 182     }
 183
 184   /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
 185      below must be changed to output_move_neon (which will use the
 186      element/structure loads/stores), and the constraint changed to 'Um' instead
 187      of 'Uv'.  */
 188
 189   switch (which_alternative)
 190     {
 191     case 0: return "vmov\t%P0, %P1  @ <mode>";
 192     case 1: case 3: return output_move_vfp (operands);
 193     case 2: gcc_unreachable ();
 194     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
 195     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
 196     default: return output_move_double (operands, true, NULL);
 197     }
 198 }
 199  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 200   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
 201   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 202   (set_attr "length" "4,4,4,4,4,4,8,8,8")
 203   (set_attr "pool_range"     "*,*,*,1020,*,*,*,1020,*")
 204   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
 205
 206 (define_insn "*neon_mov<mode>"
 207   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
 208           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
 209         (match_operand:VQXMOV 1 "general_operand"
 210           " w,w, Dn,Uni, w, r, r, Usi, r"))]
 211   "TARGET_NEON
 212    && (register_operand (operands[0], <MODE>mode)
 213        || register_operand (operands[1], <MODE>mode))"
 214 {
 215   if (which_alternative == 2)
 216     {
 217       int width, is_valid;
 218       static char templ[40];
 219
 220       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 221         &operands[1], &width);
 222
 223       gcc_assert (is_valid != 0);
 224
 225       if (width == 0)
 226         return "vmov.f32\t%q0, %1  @ <mode>";
 227       else
 228         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
 229
 230       return templ;
 231     }
 232
 233   switch (which_alternative)
 234     {
 235     case 0: return "vmov\t%q0, %q1  @ <mode>";
 236     case 1: case 3: return output_move_neon (operands);
 237     case 2: gcc_unreachable ();
 238     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 239     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 240     default: return output_move_quad (operands);
 241     }
 242 }
 243   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 244                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 245    (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
 246    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 247    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 248    (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
 249    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 250
 251 (define_expand "movti"
 252   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 253         (match_operand:TI 1 "general_operand" ""))]
 254   "TARGET_NEON"
 255 {
 256   if (can_create_pseudo_p ())
 257     {
 258       if (!REG_P (operands[0]))
 259         operands[1] = force_reg (TImode, operands[1]);
 260     }
 261 })
 262
 263 (define_expand "mov<mode>"
 264   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 265         (match_operand:VSTRUCT 1 "general_operand" ""))]
 266   "TARGET_NEON"
 267 {
 268   if (can_create_pseudo_p ())
 269     {
 270       if (!REG_P (operands[0]))
 271         operands[1] = force_reg (<MODE>mode, operands[1]);
 272     }
 273 })
 274
 275 (define_insn "*neon_mov<mode>"
 276   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 277         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 278   "TARGET_NEON
 279    && (register_operand (operands[0], <MODE>mode)
 280        || register_operand (operands[1], <MODE>mode))"
 281 {
 282   switch (which_alternative)
 283     {
 284     case 0: return "#";
 285     case 1: case 2: return output_move_neon (operands);
 286     default: gcc_unreachable ();
 287     }
 288 }
 289   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 290    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 291
 292 (define_split
 293   [(set (match_operand:EI 0 "s_register_operand" "")
 294         (match_operand:EI 1 "s_register_operand" ""))]
 295   "TARGET_NEON && reload_completed"
 296   [(set (match_dup 0) (match_dup 1))
 297    (set (match_dup 2) (match_dup 3))]
 298 {
 299   int rdest = REGNO (operands[0]);
 300   int rsrc = REGNO (operands[1]);
 301   rtx dest[2], src[2];
 302
 303   dest[0] = gen_rtx_REG (TImode, rdest);
 304   src[0] = gen_rtx_REG (TImode, rsrc);
 305   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 306   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 307
 308   neon_disambiguate_copy (operands, dest, src, 2);
 309 })
 310
 311 (define_split
 312   [(set (match_operand:OI 0 "s_register_operand" "")
 313         (match_operand:OI 1 "s_register_operand" ""))]
 314   "TARGET_NEON && reload_completed"
 315   [(set (match_dup 0) (match_dup 1))
 316    (set (match_dup 2) (match_dup 3))]
 317 {
 318   int rdest = REGNO (operands[0]);
 319   int rsrc = REGNO (operands[1]);
 320   rtx dest[2], src[2];
 321
 322   dest[0] = gen_rtx_REG (TImode, rdest);
 323   src[0] = gen_rtx_REG (TImode, rsrc);
 324   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 325   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 326
 327   neon_disambiguate_copy (operands, dest, src, 2);
 328 })
 329
 330 (define_split
 331   [(set (match_operand:CI 0 "s_register_operand" "")
 332         (match_operand:CI 1 "s_register_operand" ""))]
 333   "TARGET_NEON && reload_completed"
 334   [(set (match_dup 0) (match_dup 1))
 335    (set (match_dup 2) (match_dup 3))
 336    (set (match_dup 4) (match_dup 5))]
 337 {
 338   int rdest = REGNO (operands[0]);
 339   int rsrc = REGNO (operands[1]);
 340   rtx dest[3], src[3];
 341
 342   dest[0] = gen_rtx_REG (TImode, rdest);
 343   src[0] = gen_rtx_REG (TImode, rsrc);
 344   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 345   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 346   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 347   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 348
 349   neon_disambiguate_copy (operands, dest, src, 3);
 350 })
 351
 352 (define_split
 353   [(set (match_operand:XI 0 "s_register_operand" "")
 354         (match_operand:XI 1 "s_register_operand" ""))]
 355   "TARGET_NEON && reload_completed"
 356   [(set (match_dup 0) (match_dup 1))
 357    (set (match_dup 2) (match_dup 3))
 358    (set (match_dup 4) (match_dup 5))
 359    (set (match_dup 6) (match_dup 7))]
 360 {
 361   int rdest = REGNO (operands[0]);
 362   int rsrc = REGNO (operands[1]);
 363   rtx dest[4], src[4];
 364
 365   dest[0] = gen_rtx_REG (TImode, rdest);
 366   src[0] = gen_rtx_REG (TImode, rsrc);
 367   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 368   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 369   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 370   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 371   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 372   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 373
 374   neon_disambiguate_copy (operands, dest, src, 4);
 375 })
 376
 377 (define_expand "movmisalign<mode>"
 378   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 379         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 380                      UNSPEC_MISALIGNED_ACCESS))]
 381   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 382 {
 383   /* This pattern is not permitted to fail during expansion: if both arguments
 384      are non-registers (e.g. memory := constant, which can be created by the
 385      auto-vectorizer), force operand 1 into a register.  */
 386   if (!s_register_operand (operands[0], <MODE>mode)
 387       && !s_register_operand (operands[1], <MODE>mode))
 388     operands[1] = force_reg (<MODE>mode, operands[1]);
 389 })
 390
 391 (define_insn "*movmisalign<mode>_neon_store"
 392   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 393         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 394                     UNSPEC_MISALIGNED_ACCESS))]
 395   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 396   "vst1.<V_sz_elem>\t{%P1}, %A0"
 397   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 398
 399 (define_insn "*movmisalign<mode>_neon_load"
 400   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 401         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 402                     UNSPEC_MISALIGNED_ACCESS))]
 403   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 404   "vld1.<V_sz_elem>\t{%P0}, %A1"
 405   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 406
 407 (define_insn "*movmisalign<mode>_neon_store"
 408   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 409         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 410                     UNSPEC_MISALIGNED_ACCESS))]
 411   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 412   "vst1.<V_sz_elem>\t{%q1}, %A0"
 413   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 414
 415 (define_insn "*movmisalign<mode>_neon_load"
 416   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 417         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 418                     UNSPEC_MISALIGNED_ACCESS))]
 419   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 420   "vld1.<V_sz_elem>\t{%q0}, %A1"
 421   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 422
 423 (define_insn "vec_set<mode>_internal"
 424   [(set (match_operand:VD 0 "s_register_operand" "=w")
 425         (vec_merge:VD
 426           (vec_duplicate:VD
 427             (match_operand:<V_elem> 1 "s_register_operand" "r"))
 428           (match_operand:VD 3 "s_register_operand" "0")
 429           (match_operand:SI 2 "immediate_operand" "i")))]
 430   "TARGET_NEON"
 431 {
 432   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 433   if (BYTES_BIG_ENDIAN)
 434     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 435   operands[2] = GEN_INT (elt);
 436
 437   return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 438 }
 439   [(set_attr "neon_type" "neon_mcr")])
 440
 441 (define_insn "vec_set<mode>_internal"
 442   [(set (match_operand:VQ 0 "s_register_operand" "=w")
 443         (vec_merge:VQ
 444           (vec_duplicate:VQ
 445             (match_operand:<V_elem> 1 "s_register_operand" "r"))
 446           (match_operand:VQ 3 "s_register_operand" "0")
 447           (match_operand:SI 2 "immediate_operand" "i")))]
 448   "TARGET_NEON"
 449 {
 450   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 451   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 452   int elt = elem % half_elts;
 453   int hi = (elem / half_elts) * 2;
 454   int regno = REGNO (operands[0]);
 455
 456   if (BYTES_BIG_ENDIAN)
 457     elt = half_elts - 1 - elt;
 458
 459   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 460   operands[2] = GEN_INT (elt);
 461
 462   return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 463 }
 464   [(set_attr "neon_type" "neon_mcr")]
 465 )
 466
 467 (define_insn "vec_setv2di_internal"
 468   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
 469         (vec_merge:V2DI
 470           (vec_duplicate:V2DI
 471             (match_operand:DI 1 "s_register_operand" "r"))
 472           (match_operand:V2DI 3 "s_register_operand" "0")
 473           (match_operand:SI 2 "immediate_operand" "i")))]
 474   "TARGET_NEON"
 475 {
 476   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 477   int regno = REGNO (operands[0]) + 2 * elem;
 478
 479   operands[0] = gen_rtx_REG (DImode, regno);
 480
 481   return "vmov\t%P0, %Q1, %R1";
 482 }
 483   [(set_attr "neon_type" "neon_mcr_2_mcrr")]
 484 )
 485
 486 (define_expand "vec_set<mode>"
 487   [(match_operand:VDQ 0 "s_register_operand" "")
 488    (match_operand:<V_elem> 1 "s_register_operand" "")
 489    (match_operand:SI 2 "immediate_operand" "")]
 490   "TARGET_NEON"
 491 {
 492   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 493   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 494                                          GEN_INT (elem), operands[0]));
 495   DONE;
 496 })
 497
 498 (define_insn "vec_extract<mode>"
 499   [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
 500         (vec_select:<V_elem>
 501           (match_operand:VD 1 "s_register_operand" "w")
 502           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 503   "TARGET_NEON"
 504 {
 505   if (BYTES_BIG_ENDIAN)
 506     {
 507       int elt = INTVAL (operands[2]);
 508       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 509       operands[2] = GEN_INT (elt);
 510     }
 511   return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 512 }
 513   [(set_attr "neon_type" "neon_bp_simple")]
 514 )
 515
 516 (define_insn "vec_extract<mode>"
 517   [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
 518         (vec_select:<V_elem>
 519           (match_operand:VQ 1 "s_register_operand" "w")
 520           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 521   "TARGET_NEON"
 522 {
 523   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 524   int elt = INTVAL (operands[2]) % half_elts;
 525   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 526   int regno = REGNO (operands[1]);
 527
 528   if (BYTES_BIG_ENDIAN)
 529     elt = half_elts - 1 - elt;
 530
 531   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 532   operands[2] = GEN_INT (elt);
 533
 534   return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 535 }
 536   [(set_attr "neon_type" "neon_bp_simple")]
 537 )
 538
 539 (define_insn "vec_extractv2di"
 540   [(set (match_operand:DI 0 "s_register_operand" "=r")
 541         (vec_select:DI
 542           (match_operand:V2DI 1 "s_register_operand" "w")
 543           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 544   "TARGET_NEON"
 545 {
 546   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 547
 548   operands[1] = gen_rtx_REG (DImode, regno);
 549
 550   return "vmov\t%Q0, %R0, %P1  @ v2di";
 551 }
 552   [(set_attr "neon_type" "neon_int_1")]
 553 )
 554
 555 (define_expand "vec_init<mode>"
 556   [(match_operand:VDQ 0 "s_register_operand" "")
 557    (match_operand 1 "" "")]
 558   "TARGET_NEON"
 559 {
 560   neon_expand_vector_init (operands[0], operands[1]);
 561   DONE;
 562 })
 563
 564 ;; Doubleword and quadword arithmetic.
 565
 566 ;; NOTE: some other instructions also support 64-bit integer
 567 ;; element size, which we could potentially use for "long long" operations.
 568
 569 (define_insn "*add<mode>3_neon"
 570   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 571         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 572                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 573   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 574   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 575   [(set (attr "neon_type")
 576       (if_then_else (match_test "<Is_float_mode>")
 577                     (if_then_else (match_test "<Is_d_reg>")
 578                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 579                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 580                     (const_string "neon_int_1")))]
 581 )
 582
 583 (define_insn "adddi3_neon"
 584   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 585         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 586                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 587    (clobber (reg:CC CC_REGNUM))]
 588   "TARGET_NEON"
 589 {
 590   switch (which_alternative)
 591     {
 592     case 0: /* fall through */
 593     case 3: return "vadd.i64\t%P0, %P1, %P2";
 594     case 1: return "#";
 595     case 2: return "#";
 596     case 4: return "#";
 597     case 5: return "#";
 598     case 6: return "#";
 599     default: gcc_unreachable ();
 600     }
 601 }
 602   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 603    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 604    (set_attr "length" "*,8,8,*,8,8,8")
 605    (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
 606 )
 607
 608 (define_insn "*sub<mode>3_neon"
 609   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 610         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 611                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 612   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 613   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 614   [(set (attr "neon_type")
 615       (if_then_else (match_test "<Is_float_mode>")
 616                     (if_then_else (match_test "<Is_d_reg>")
 617                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 618                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 619                     (const_string "neon_int_2")))]
 620 )
 621
 622 (define_insn "subdi3_neon"
 623   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 624         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 625                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 626    (clobber (reg:CC CC_REGNUM))]
 627   "TARGET_NEON"
 628 {
 629   switch (which_alternative)
 630     {
 631     case 0: /* fall through */
 632     case 4: return "vsub.i64\t%P0, %P1, %P2";
 633     case 1: /* fall through */
 634     case 2: /* fall through */
 635     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 636     default: gcc_unreachable ();
 637     }
 638 }
 639   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 640    (set_attr "conds" "*,clob,clob,clob,*")
 641    (set_attr "length" "*,8,8,8,*")
 642    (set_attr "arch" "nota8,*,*,*,onlya8")]
 643 )
 644
 645 (define_insn "*mul<mode>3_neon"
 646   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 647         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 648                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 649   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 650   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 651   [(set (attr "neon_type")
 652       (if_then_else (match_test "<Is_float_mode>")
 653                     (if_then_else (match_test "<Is_d_reg>")
 654                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 655                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 656                     (if_then_else (match_test "<Is_d_reg>")
 657                                   (if_then_else
 658                                     (match_test "<Scalar_mul_8_16>")
 659                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 660                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 661                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 662                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 663                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 664 )
 665
 666 (define_insn "mul<mode>3add<mode>_neon"
 667   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 668         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 669                             (match_operand:VDQ 3 "s_register_operand" "w"))
 670                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 671   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 672   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 673   [(set (attr "neon_type")
 674       (if_then_else (match_test "<Is_float_mode>")
 675                     (if_then_else (match_test "<Is_d_reg>")
 676                                   (const_string "neon_fp_vmla_ddd")
 677                                   (const_string "neon_fp_vmla_qqq"))
 678                     (if_then_else (match_test "<Is_d_reg>")
 679                                   (if_then_else
 680                                     (match_test "<Scalar_mul_8_16>")
 681                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 682                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 683                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 684                                     (const_string "neon_mla_qqq_8_16")
 685                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 686 )
 687
 688 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 689   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 690         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 691                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 692                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 693   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 694   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 695   [(set (attr "neon_type")
 696       (if_then_else (match_test "<Is_float_mode>")
 697                     (if_then_else (match_test "<Is_d_reg>")
 698                                   (const_string "neon_fp_vmla_ddd")
 699                                   (const_string "neon_fp_vmla_qqq"))
 700                     (if_then_else (match_test "<Is_d_reg>")
 701                                   (if_then_else
 702                                     (match_test "<Scalar_mul_8_16>")
 703                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 704                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 705                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 706                                     (const_string "neon_mla_qqq_8_16")
 707                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 708 )
 709
 710 (define_insn "ior<mode>3"
 711   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 712         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 713                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 714   "TARGET_NEON"
 715 {
 716   switch (which_alternative)
 717     {
 718     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 719     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 720                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 721     default: gcc_unreachable ();
 722     }
 723 }
 724   [(set_attr "neon_type" "neon_int_1")]
 725 )
 726
 727 (define_insn "iordi3_neon"
 728   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 729         (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 730                 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
 731   "TARGET_NEON"
 732 {
 733   switch (which_alternative)
 734     {
 735     case 0: /* fall through */
 736     case 4: return "vorr\t%P0, %P1, %P2";
 737     case 1: /* fall through */
 738     case 5: return neon_output_logic_immediate ("vorr", &operands[2],
 739                      DImode, 0, VALID_NEON_QREG_MODE (DImode));
 740     case 2: return "#";
 741     case 3: return "#";
 742     default: gcc_unreachable ();
 743     }
 744 }
 745   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 746    (set_attr "length" "*,*,8,8,*,*")
 747    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 748 )
 749
 750 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 751 ;; vorr. We support the pseudo-instruction vand instead, because that
 752 ;; corresponds to the canonical form the middle-end expects to use for
 753 ;; immediate bitwise-ANDs.
 754
 755 (define_insn "and<mode>3"
 756   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 757         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 758                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 759   "TARGET_NEON"
 760 {
 761   switch (which_alternative)
 762     {
 763     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 764     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 765                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 766     default: gcc_unreachable ();
 767     }
 768 }
 769   [(set_attr "neon_type" "neon_int_1")]
 770 )
 771
 772 (define_insn "anddi3_neon"
 773   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 774         (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 775                 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
 776   "TARGET_NEON"
 777 {
 778   switch (which_alternative)
 779     {
 780     case 0: /* fall through */
 781     case 4: return "vand\t%P0, %P1, %P2";
 782     case 1: /* fall through */
 783     case 5: return neon_output_logic_immediate ("vand", &operands[2],
 784                      DImode, 1, VALID_NEON_QREG_MODE (DImode));
 785     case 2: return "#";
 786     case 3: return "#";
 787     default: gcc_unreachable ();
 788     }
 789 }
 790   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 791    (set_attr "length" "*,*,8,8,*,*")
 792    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 793 )
 794
 795 (define_insn "orn<mode>3_neon"
 796   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 797         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 798                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 799   "TARGET_NEON"
 800   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 801   [(set_attr "neon_type" "neon_int_1")]
 802 )
 803
 804 ;; TODO: investigate whether we should disable
 805 ;; this and bicdi3_neon for the A8 in line with the other
 806 ;; changes above.
 807 (define_insn_and_split "orndi3_neon"
 808   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 809         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 810                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 811   "TARGET_NEON"
 812   "@
 813    vorn\t%P0, %P1, %P2
 814    #
 815    #
 816    #"
 817   "reload_completed &&
 818    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 819   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 820    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 821   "
 822   {
 823     if (TARGET_THUMB2)
 824       {
 825         operands[3] = gen_highpart (SImode, operands[0]);
 826         operands[0] = gen_lowpart (SImode, operands[0]);
 827         operands[4] = gen_highpart (SImode, operands[2]);
 828         operands[2] = gen_lowpart (SImode, operands[2]);
 829         operands[5] = gen_highpart (SImode, operands[1]);
 830         operands[1] = gen_lowpart (SImode, operands[1]);
 831       }
 832     else
 833       {
 834         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 835         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 836         DONE;
 837       }
 838   }"
 839   [(set_attr "neon_type" "neon_int_1,*,*,*")
 840    (set_attr "length" "*,16,8,8")
 841    (set_attr "arch" "any,a,t2,t2")]
 842 )
 843
 844 (define_insn "bic<mode>3_neon"
 845   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 846         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 847                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 848   "TARGET_NEON"
 849   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 850   [(set_attr "neon_type" "neon_int_1")]
 851 )
 852
 853 ;; Compare to *anddi_notdi_di.
 854 (define_insn "bicdi3_neon"
 855   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 856         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 857                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 858   "TARGET_NEON"
 859   "@
 860    vbic\t%P0, %P1, %P2
 861    #
 862    #"
 863   [(set_attr "neon_type" "neon_int_1,*,*")
 864    (set_attr "length" "*,8,8")]
 865 )
 866
 867 (define_insn "xor<mode>3"
 868   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 869         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 870                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 871   "TARGET_NEON"
 872   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 873   [(set_attr "neon_type" "neon_int_1")]
 874 )
 875
 876 (define_insn "xordi3_neon"
 877   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
 878         (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
 879                 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
 880   "TARGET_NEON"
 881   "@
 882    veor\t%P0, %P1, %P2
 883    #
 884    #
 885    veor\t%P0, %P1, %P2"
 886   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
 887    (set_attr "length" "*,8,8,*")
 888    (set_attr "arch" "nota8,*,*,onlya8")]
 889 )
 890
 891 (define_insn "one_cmpl<mode>2"
 892   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 893         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 894   "TARGET_NEON"
 895   "vmvn\t%<V_reg>0, %<V_reg>1"
 896   [(set_attr "neon_type" "neon_int_1")]
 897 )
 898
 899 (define_insn "abs<mode>2"
 900   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 901         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 902   "TARGET_NEON"
 903   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 904   [(set (attr "neon_type")
 905       (if_then_else (match_test "<Is_float_mode>")
 906                     (if_then_else (match_test "<Is_d_reg>")
 907                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 908                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 909                     (const_string "neon_int_3")))]
 910 )
 911
 912 (define_insn "neg<mode>2"
 913   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 914         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 915   "TARGET_NEON"
 916   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 917   [(set (attr "neon_type")
 918       (if_then_else (match_test "<Is_float_mode>")
 919                     (if_then_else (match_test "<Is_d_reg>")
 920                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 921                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 922                     (const_string "neon_int_3")))]
 923 )
 924
 925 (define_insn "negdi2_neon"
 926   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 927         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 928    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 929    (clobber (reg:CC CC_REGNUM))]
 930   "TARGET_NEON"
 931   "#"
 932   [(set_attr "length" "8")]
 933 )
 934
 935 ; Split negdi2_neon for vfp registers
 936 (define_split
 937   [(set (match_operand:DI 0 "s_register_operand" "")
 938         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 939    (clobber (match_scratch:DI 2 ""))
 940    (clobber (reg:CC CC_REGNUM))]
 941   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 942   [(set (match_dup 2) (const_int 0))
 943    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 944               (clobber (reg:CC CC_REGNUM))])]
 945   {
 946     if (!REG_P (operands[2]))
 947       operands[2] = operands[0];
 948   }
 949 )
 950
 951 ; Split negdi2_neon for core registers
 952 (define_split
 953   [(set (match_operand:DI 0 "s_register_operand" "")
 954         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 955    (clobber (match_scratch:DI 2 ""))
 956    (clobber (reg:CC CC_REGNUM))]
 957   "TARGET_32BIT && reload_completed
 958    && arm_general_register_operand (operands[0], DImode)"
 959   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 960               (clobber (reg:CC CC_REGNUM))])]
 961   ""
 962 )
 963
 964 (define_insn "*umin<mode>3_neon"
 965   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 966         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 967                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 968   "TARGET_NEON"
 969   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 970   [(set_attr "neon_type" "neon_int_5")]
 971 )
 972
 973 (define_insn "*umax<mode>3_neon"
 974   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 975         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 976                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 977   "TARGET_NEON"
 978   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 979   [(set_attr "neon_type" "neon_int_5")]
 980 )
 981
 982 (define_insn "*smin<mode>3_neon"
 983   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 984         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 985                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 986   "TARGET_NEON"
 987   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 988   [(set (attr "neon_type")
 989       (if_then_else (match_test "<Is_float_mode>")
 990                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 991                     (const_string "neon_int_5")))]
 992 )
 993
 994 (define_insn "*smax<mode>3_neon"
 995   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 996         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 997                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 998   "TARGET_NEON"
 999   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1000   [(set (attr "neon_type")
1001       (if_then_else (match_test "<Is_float_mode>")
1002                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1003                     (const_string "neon_int_5")))]
1004 )
1005
1006 ; TODO: V2DI shifts are current disabled because there are bugs in the
1007 ; generic vectorizer code.  It ends up creating a V2DI constructor with
1008 ; SImode elements.
1009
1010 (define_insn "vashl<mode>3"
1011   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
1012         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
1013                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
1014   "TARGET_NEON"
1015   {
1016     switch (which_alternative)
1017       {
1018         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
1019         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
1020                                                     <MODE>mode,
1021                                                     VALID_NEON_QREG_MODE (<MODE>mode),
1022                                                     true);
1023         default: gcc_unreachable ();
1024       }
1025   }
1026   [(set (attr "neon_type")
1027       (if_then_else (match_test "<Is_d_reg>")
1028                     (const_string "neon_vshl_ddd")
1029                     (const_string "neon_shift_3")))]
1030 )
1031
1032 (define_insn "vashr<mode>3_imm"
1033   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1034         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1035                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1036   "TARGET_NEON"
1037   {
1038     return neon_output_shift_immediate ("vshr", 's', &operands[2],
1039                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1040                                         false);
1041   }
1042   [(set (attr "neon_type")
1043       (if_then_else (match_test "<Is_d_reg>")
1044                     (const_string "neon_vshl_ddd")
1045                     (const_string "neon_shift_3")))]
1046 )
1047
1048 (define_insn "vlshr<mode>3_imm"
1049   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1050         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1051                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1052   "TARGET_NEON"
1053   {
1054     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1055                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1056                                         false);
1057   }
1058   [(set (attr "neon_type")
1059         (if_then_else (match_test "<Is_d_reg>")
1060                       (const_string "neon_vshl_ddd")
1061                       (const_string "neon_shift_3")))]
1062 )
1063
1064 ; Used for implementing logical shift-right, which is a left-shift by a negative
1065 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1066 ; above, but using an unspec in case GCC tries anything tricky with negative
1067 ; shift amounts.
1068
1069 (define_insn "ashl<mode>3_signed"
1070   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1071         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1072                       (match_operand:VDQI 2 "s_register_operand" "w")]
1073                      UNSPEC_ASHIFT_SIGNED))]
1074   "TARGET_NEON"
1075   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1076   [(set (attr "neon_type")
1077       (if_then_else (match_test "<Is_d_reg>")
1078                     (const_string "neon_vshl_ddd")
1079                     (const_string "neon_shift_3")))]
1080 )
1081
1082 ; Used for implementing logical shift-right, which is a left-shift by a negative
1083 ; amount, with unsigned operands.
1084
1085 (define_insn "ashl<mode>3_unsigned"
1086   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1087         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1088                       (match_operand:VDQI 2 "s_register_operand" "w")]
1089                      UNSPEC_ASHIFT_UNSIGNED))]
1090   "TARGET_NEON"
1091   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1092   [(set (attr "neon_type")
1093       (if_then_else (match_test "<Is_d_reg>")
1094                     (const_string "neon_vshl_ddd")
1095                     (const_string "neon_shift_3")))]
1096 )
1097
1098 (define_expand "vashr<mode>3"
1099   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1100         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1101                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1102   "TARGET_NEON"
1103 {
1104   if (s_register_operand (operands[2], <MODE>mode))
1105     {
1106       rtx neg = gen_reg_rtx (<MODE>mode);
1107       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1108       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1109     }
1110   else
1111     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1112   DONE;
1113 })
1114
1115 (define_expand "vlshr<mode>3"
1116   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1117         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1118                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1119   "TARGET_NEON"
1120 {
1121   if (s_register_operand (operands[2], <MODE>mode))
1122     {
1123       rtx neg = gen_reg_rtx (<MODE>mode);
1124       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1125       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1126     }
1127   else
1128     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1129   DONE;
1130 })
1131
1132 ;; Widening operations
1133
1134 (define_insn "widen_ssum<mode>3"
1135   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1136         (plus:<V_widen> (sign_extend:<V_widen>
1137                           (match_operand:VW 1 "s_register_operand" "%w"))
1138                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1139   "TARGET_NEON"
1140   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1141   [(set_attr "neon_type" "neon_int_3")]
1142 )
1143
1144 (define_insn "widen_usum<mode>3"
1145   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1146         (plus:<V_widen> (zero_extend:<V_widen>
1147                           (match_operand:VW 1 "s_register_operand" "%w"))
1148                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1149   "TARGET_NEON"
1150   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1151   [(set_attr "neon_type" "neon_int_3")]
1152 )
1153
1154 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1155 ;; shift-count granularity. That's good enough for the middle-end's current
1156 ;; needs.
1157
1158 ;; Note that it's not safe to perform such an operation in big-endian mode,
1159 ;; due to element-ordering issues.
1160
1161 (define_expand "vec_shr_<mode>"
1162   [(match_operand:VDQ 0 "s_register_operand" "")
1163    (match_operand:VDQ 1 "s_register_operand" "")
1164    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1165   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1166 {
1167   rtx zero_reg;
1168   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1169   const int width = GET_MODE_BITSIZE (<MODE>mode);
1170   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1171   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1172     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1173
1174   if (num_bits == width)
1175     {
1176       emit_move_insn (operands[0], operands[1]);
1177       DONE;
1178     }
1179
1180   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1181   operands[0] = gen_lowpart (bvecmode, operands[0]);
1182   operands[1] = gen_lowpart (bvecmode, operands[1]);
1183
1184   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1185                       GEN_INT (num_bits / BITS_PER_UNIT)));
1186   DONE;
1187 })
1188
1189 (define_expand "vec_shl_<mode>"
1190   [(match_operand:VDQ 0 "s_register_operand" "")
1191    (match_operand:VDQ 1 "s_register_operand" "")
1192    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1193   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1194 {
1195   rtx zero_reg;
1196   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1197   const int width = GET_MODE_BITSIZE (<MODE>mode);
1198   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1199   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1200     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1201
1202   if (num_bits == 0)
1203     {
1204       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1205       DONE;
1206     }
1207
1208   num_bits = width - num_bits;
1209
1210   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1211   operands[0] = gen_lowpart (bvecmode, operands[0]);
1212   operands[1] = gen_lowpart (bvecmode, operands[1]);
1213
1214   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1215                       GEN_INT (num_bits / BITS_PER_UNIT)));
1216   DONE;
1217 })
1218
1219 ;; Helpers for quad-word reduction operations
1220
1221 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1222 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1223 ; N/2-element vector.
1224
1225 (define_insn "quad_halves_<code>v4si"
1226   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1227         (vqh_ops:V2SI
1228           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1229                            (parallel [(const_int 0) (const_int 1)]))
1230           (vec_select:V2SI (match_dup 1)
1231                            (parallel [(const_int 2) (const_int 3)]))))]
1232   "TARGET_NEON"
1233   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1234   [(set_attr "vqh_mnem" "<VQH_mnem>")
1235    (set (attr "neon_type")
1236       (if_then_else (eq_attr "vqh_mnem" "vadd")
1237                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1238 )
1239
1240 (define_insn "quad_halves_<code>v4sf"
1241   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1242         (vqhs_ops:V2SF
1243           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1244                            (parallel [(const_int 0) (const_int 1)]))
1245           (vec_select:V2SF (match_dup 1)
1246                            (parallel [(const_int 2) (const_int 3)]))))]
1247   "TARGET_NEON && flag_unsafe_math_optimizations"
1248   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1249   [(set_attr "vqh_mnem" "<VQH_mnem>")
1250    (set (attr "neon_type")
1251       (if_then_else (eq_attr "vqh_mnem" "vadd")
1252                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1253 )
1254
1255 (define_insn "quad_halves_<code>v8hi"
1256   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1257         (vqh_ops:V4HI
1258           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1259                            (parallel [(const_int 0) (const_int 1)
1260                                       (const_int 2) (const_int 3)]))
1261           (vec_select:V4HI (match_dup 1)
1262                            (parallel [(const_int 4) (const_int 5)
1263                                       (const_int 6) (const_int 7)]))))]
1264   "TARGET_NEON"
1265   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1266   [(set_attr "vqh_mnem" "<VQH_mnem>")
1267    (set (attr "neon_type")
1268       (if_then_else (eq_attr "vqh_mnem" "vadd")
1269                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1270 )
1271
1272 (define_insn "quad_halves_<code>v16qi"
1273   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1274         (vqh_ops:V8QI
1275           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1276                            (parallel [(const_int 0) (const_int 1)
1277                                       (const_int 2) (const_int 3)
1278                                       (const_int 4) (const_int 5)
1279                                       (const_int 6) (const_int 7)]))
1280           (vec_select:V8QI (match_dup 1)
1281                            (parallel [(const_int 8) (const_int 9)
1282                                       (const_int 10) (const_int 11)
1283                                       (const_int 12) (const_int 13)
1284                                       (const_int 14) (const_int 15)]))))]
1285   "TARGET_NEON"
1286   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1287   [(set_attr "vqh_mnem" "<VQH_mnem>")
1288    (set (attr "neon_type")
1289       (if_then_else (eq_attr "vqh_mnem" "vadd")
1290                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1291 )
1292
1293 (define_expand "move_hi_quad_<mode>"
1294  [(match_operand:ANY128 0 "s_register_operand" "")
1295   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1296  "TARGET_NEON"
1297 {
1298   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1299                                        GET_MODE_SIZE (<V_HALF>mode)),
1300                   operands[1]);
1301   DONE;
1302 })
1303
1304 (define_expand "move_lo_quad_<mode>"
1305  [(match_operand:ANY128 0 "s_register_operand" "")
1306   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1307  "TARGET_NEON"
1308 {
1309   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1310                                        <MODE>mode, 0),
1311                   operands[1]);
1312   DONE;
1313 })
1314
1315 ;; Reduction operations
1316
1317 (define_expand "reduc_splus_<mode>"
1318   [(match_operand:VD 0 "s_register_operand" "")
1319    (match_operand:VD 1 "s_register_operand" "")]
1320   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1321 {
1322   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1323                         &gen_neon_vpadd_internal<mode>);
1324   DONE;
1325 })
1326
1327 (define_expand "reduc_splus_<mode>"
1328   [(match_operand:VQ 0 "s_register_operand" "")
1329    (match_operand:VQ 1 "s_register_operand" "")]
1330   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1331    && !BYTES_BIG_ENDIAN"
1332 {
1333   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1334   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1335
1336   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1337   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1338   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1339
1340   DONE;
1341 })
1342
1343 (define_insn "reduc_splus_v2di"
1344   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1345         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1346                      UNSPEC_VPADD))]
1347   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1348   "vadd.i64\t%e0, %e1, %f1"
1349   [(set_attr "neon_type" "neon_int_1")]
1350 )
1351
1352 ;; NEON does not distinguish between signed and unsigned addition except on
1353 ;; widening operations.
1354 (define_expand "reduc_uplus_<mode>"
1355   [(match_operand:VDQI 0 "s_register_operand" "")
1356    (match_operand:VDQI 1 "s_register_operand" "")]
1357   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1358 {
1359   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1360   DONE;
1361 })
1362
1363 (define_expand "reduc_smin_<mode>"
1364   [(match_operand:VD 0 "s_register_operand" "")
1365    (match_operand:VD 1 "s_register_operand" "")]
1366   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1367 {
1368   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1369                         &gen_neon_vpsmin<mode>);
1370   DONE;
1371 })
1372
1373 (define_expand "reduc_smin_<mode>"
1374   [(match_operand:VQ 0 "s_register_operand" "")
1375    (match_operand:VQ 1 "s_register_operand" "")]
1376   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1377    && !BYTES_BIG_ENDIAN"
1378 {
1379   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1380   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1381
1382   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1383   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1384   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1385
1386   DONE;
1387 })
1388
1389 (define_expand "reduc_smax_<mode>"
1390   [(match_operand:VD 0 "s_register_operand" "")
1391    (match_operand:VD 1 "s_register_operand" "")]
1392   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1393 {
1394   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1395                         &gen_neon_vpsmax<mode>);
1396   DONE;
1397 })
1398
1399 (define_expand "reduc_smax_<mode>"
1400   [(match_operand:VQ 0 "s_register_operand" "")
1401    (match_operand:VQ 1 "s_register_operand" "")]
1402   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1403    && !BYTES_BIG_ENDIAN"
1404 {
1405   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1406   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1407
1408   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1409   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1410   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1411
1412   DONE;
1413 })
1414
1415 (define_expand "reduc_umin_<mode>"
1416   [(match_operand:VDI 0 "s_register_operand" "")
1417    (match_operand:VDI 1 "s_register_operand" "")]
1418   "TARGET_NEON"
1419 {
1420   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1421                         &gen_neon_vpumin<mode>);
1422   DONE;
1423 })
1424
1425 (define_expand "reduc_umin_<mode>"
1426   [(match_operand:VQI 0 "s_register_operand" "")
1427    (match_operand:VQI 1 "s_register_operand" "")]
1428   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1429 {
1430   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1431   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1432
1433   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1434   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1435   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1436
1437   DONE;
1438 })
1439
1440 (define_expand "reduc_umax_<mode>"
1441   [(match_operand:VDI 0 "s_register_operand" "")
1442    (match_operand:VDI 1 "s_register_operand" "")]
1443   "TARGET_NEON"
1444 {
1445   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1446                         &gen_neon_vpumax<mode>);
1447   DONE;
1448 })
1449
1450 (define_expand "reduc_umax_<mode>"
1451   [(match_operand:VQI 0 "s_register_operand" "")
1452    (match_operand:VQI 1 "s_register_operand" "")]
1453   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1454 {
1455   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1456   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1457
1458   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1459   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1460   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1461
1462   DONE;
1463 })
1464
1465 (define_insn "neon_vpadd_internal<mode>"
1466   [(set (match_operand:VD 0 "s_register_operand" "=w")
1467         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1468                     (match_operand:VD 2 "s_register_operand" "w")]
1469                    UNSPEC_VPADD))]
1470   "TARGET_NEON"
1471   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1472   ;; Assume this schedules like vadd.
1473   [(set (attr "neon_type")
1474       (if_then_else (match_test "<Is_float_mode>")
1475                     (if_then_else (match_test "<Is_d_reg>")
1476                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1477                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1478                     (const_string "neon_int_1")))]
1479 )
1480
1481 (define_insn "neon_vpsmin<mode>"
1482   [(set (match_operand:VD 0 "s_register_operand" "=w")
1483         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1484                     (match_operand:VD 2 "s_register_operand" "w")]
1485                    UNSPEC_VPSMIN))]
1486   "TARGET_NEON"
1487   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1488   ;; Assume this schedules like vmin.
1489   [(set (attr "neon_type")
1490       (if_then_else (match_test "<Is_float_mode>")
1491                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1492                     (const_string "neon_int_5")))]
1493 )
1494
1495 (define_insn "neon_vpsmax<mode>"
1496   [(set (match_operand:VD 0 "s_register_operand" "=w")
1497         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1498                     (match_operand:VD 2 "s_register_operand" "w")]
1499                    UNSPEC_VPSMAX))]
1500   "TARGET_NEON"
1501   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1502   ;; Assume this schedules like vmax.
1503   [(set (attr "neon_type")
1504       (if_then_else (match_test "<Is_float_mode>")
1505                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1506                     (const_string "neon_int_5")))]
1507 )
1508
1509 (define_insn "neon_vpumin<mode>"
1510   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1511         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1512                      (match_operand:VDI 2 "s_register_operand" "w")]
1513                    UNSPEC_VPUMIN))]
1514   "TARGET_NEON"
1515   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1516   ;; Assume this schedules like umin.
1517   [(set_attr "neon_type" "neon_int_5")]
1518 )
1519
1520 (define_insn "neon_vpumax<mode>"
1521   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1522         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1523                      (match_operand:VDI 2 "s_register_operand" "w")]
1524                    UNSPEC_VPUMAX))]
1525   "TARGET_NEON"
1526   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1527   ;; Assume this schedules like umax.
1528   [(set_attr "neon_type" "neon_int_5")]
1529 )
1530
1531 ;; Saturating arithmetic
1532
1533 ; NOTE: Neon supports many more saturating variants of instructions than the
1534 ; following, but these are all GCC currently understands.
1535 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1536 ; yet either, although these patterns may be used by intrinsics when they're
1537 ; added.
1538
1539 (define_insn "*ss_add<mode>_neon"
1540   [(set (match_operand:VD 0 "s_register_operand" "=w")
1541        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1542                    (match_operand:VD 2 "s_register_operand" "w")))]
1543   "TARGET_NEON"
1544   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1545   [(set_attr "neon_type" "neon_int_4")]
1546 )
1547
1548 (define_insn "*us_add<mode>_neon"
1549   [(set (match_operand:VD 0 "s_register_operand" "=w")
1550        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1551                    (match_operand:VD 2 "s_register_operand" "w")))]
1552   "TARGET_NEON"
1553   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1554   [(set_attr "neon_type" "neon_int_4")]
1555 )
1556
1557 (define_insn "*ss_sub<mode>_neon"
1558   [(set (match_operand:VD 0 "s_register_operand" "=w")
1559        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1560                     (match_operand:VD 2 "s_register_operand" "w")))]
1561   "TARGET_NEON"
1562   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1563   [(set_attr "neon_type" "neon_int_5")]
1564 )
1565
1566 (define_insn "*us_sub<mode>_neon"
1567   [(set (match_operand:VD 0 "s_register_operand" "=w")
1568        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1569                     (match_operand:VD 2 "s_register_operand" "w")))]
1570   "TARGET_NEON"
1571   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1572   [(set_attr "neon_type" "neon_int_5")]
1573 )
1574
1575 ;; Conditional instructions.  These are comparisons with conditional moves for
1576 ;; vectors.  They perform the assignment:
1577 ;;
1578 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1579 ;;
1580 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1581 ;; element-wise.
1582
1583 (define_expand "vcond<mode><mode>"
1584   [(set (match_operand:VDQW 0 "s_register_operand" "")
1585         (if_then_else:VDQW
1586           (match_operator 3 "arm_comparison_operator"
1587             [(match_operand:VDQW 4 "s_register_operand" "")
1588              (match_operand:VDQW 5 "nonmemory_operand" "")])
1589           (match_operand:VDQW 1 "s_register_operand" "")
1590           (match_operand:VDQW 2 "s_register_operand" "")))]
1591   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1592 {
1593   rtx mask;
1594   int inverse = 0, immediate_zero = 0;
1595   /* See the description of "magic" bits in the 'T' case of
1596      arm_print_operand.  */
1597   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1598                              ? 3 : 1;
1599   rtx magic_rtx = GEN_INT (magic_word);
1600
1601   mask = gen_reg_rtx (<V_cmp_result>mode);
1602
1603   if (operands[5] == CONST0_RTX (<MODE>mode))
1604     immediate_zero = 1;
1605   else if (!REG_P (operands[5]))
1606     operands[5] = force_reg (<MODE>mode, operands[5]);
1607
1608   switch (GET_CODE (operands[3]))
1609     {
1610     case GE:
1611       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1612                                       magic_rtx));
1613       break;
1614
1615     case GT:
1616       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1617                                       magic_rtx));
1618       break;
1619
1620     case EQ:
1621       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1622                                       magic_rtx));
1623       break;
1624
1625     case LE:
1626       if (immediate_zero)
1627         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1628                                         magic_rtx));
1629       else
1630         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1631                                         magic_rtx));
1632       break;
1633
1634     case LT:
1635       if (immediate_zero)
1636         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1637                                         magic_rtx));
1638       else
1639         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1640                                         magic_rtx));
1641       break;
1642
1643     case NE:
1644       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1645                                       magic_rtx));
1646       inverse = 1;
1647       break;
1648
1649     default:
1650       gcc_unreachable ();
1651     }
1652
1653   if (inverse)
1654     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1655                                     operands[1]));
1656   else
1657     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1658                                     operands[2]));
1659
1660   DONE;
1661 })
1662
1663 (define_expand "vcondu<mode><mode>"
1664   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1665         (if_then_else:VDQIW
1666           (match_operator 3 "arm_comparison_operator"
1667             [(match_operand:VDQIW 4 "s_register_operand" "")
1668              (match_operand:VDQIW 5 "s_register_operand" "")])
1669           (match_operand:VDQIW 1 "s_register_operand" "")
1670           (match_operand:VDQIW 2 "s_register_operand" "")))]
1671   "TARGET_NEON"
1672 {
1673   rtx mask;
1674   int inverse = 0, immediate_zero = 0;
1675
1676   mask = gen_reg_rtx (<V_cmp_result>mode);
1677
1678   if (operands[5] == CONST0_RTX (<MODE>mode))
1679     immediate_zero = 1;
1680   else if (!REG_P (operands[5]))
1681     operands[5] = force_reg (<MODE>mode, operands[5]);
1682
1683   switch (GET_CODE (operands[3]))
1684     {
1685     case GEU:
1686       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1687                                       const0_rtx));
1688       break;
1689
1690     case GTU:
1691       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1692                                       const0_rtx));
1693       break;
1694
1695     case EQ:
1696       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1697                                       const0_rtx));
1698       break;
1699
1700     case LEU:
1701       if (immediate_zero)
1702         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1703                                         const0_rtx));
1704       else
1705         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1706                                         const0_rtx));
1707       break;
1708
1709     case LTU:
1710       if (immediate_zero)
1711         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1712                                         const0_rtx));
1713       else
1714         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1715                                         const0_rtx));
1716       break;
1717
1718     case NE:
1719       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1720                                       const0_rtx));
1721       inverse = 1;
1722       break;
1723
1724     default:
1725       gcc_unreachable ();
1726     }
1727
1728   if (inverse)
1729     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1730                                     operands[1]));
1731   else
1732     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1733                                     operands[2]));
1734
1735   DONE;
1736 })
1737
1738 ;; Patterns for builtins.
1739
1740 ; good for plain vadd, vaddq.
1741
1742 (define_expand "neon_vadd<mode>"
1743   [(match_operand:VDQX 0 "s_register_operand" "=w")
1744    (match_operand:VDQX 1 "s_register_operand" "w")
1745    (match_operand:VDQX 2 "s_register_operand" "w")
1746    (match_operand:SI 3 "immediate_operand" "i")]
1747   "TARGET_NEON"
1748 {
1749   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1750     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1751   else
1752     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1753                                            operands[2]));
1754   DONE;
1755 })
1756
1757 ; Note that NEON operations don't support the full IEEE 754 standard: in
1758 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1759 ; use those instructions for autovectorization, etc. unless
1760 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1761 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1762 ; header) must work in either case: if -funsafe-math-optimizations is given,
1763 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1764 ; expand to unspecs (which may potentially limit the extent to which they might
1765 ; be optimized by generic code).
1766
1767 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1768
1769 (define_insn "neon_vadd<mode>_unspec"
1770   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1771         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1772                       (match_operand:VDQX 2 "s_register_operand" "w")]
1773                      UNSPEC_VADD))]
1774   "TARGET_NEON"
1775   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1776   [(set (attr "neon_type")
1777       (if_then_else (match_test "<Is_float_mode>")
1778                     (if_then_else (match_test "<Is_d_reg>")
1779                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1780                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1781                     (const_string "neon_int_1")))]
1782 )
1783
1784 ; operand 3 represents in bits:
1785 ;  bit 0: signed (vs unsigned).
1786 ;  bit 1: rounding (vs none).
1787
1788 (define_insn "neon_vaddl<mode>"
1789   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1790         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1791                            (match_operand:VDI 2 "s_register_operand" "w")
1792                            (match_operand:SI 3 "immediate_operand" "i")]
1793                           UNSPEC_VADDL))]
1794   "TARGET_NEON"
1795   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1796   [(set_attr "neon_type" "neon_int_3")]
1797 )
1798
1799 (define_insn "neon_vaddw<mode>"
1800   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1801         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1802                            (match_operand:VDI 2 "s_register_operand" "w")
1803                            (match_operand:SI 3 "immediate_operand" "i")]
1804                           UNSPEC_VADDW))]
1805   "TARGET_NEON"
1806   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1807   [(set_attr "neon_type" "neon_int_2")]
1808 )
1809
1810 ; vhadd and vrhadd.
1811
1812 (define_insn "neon_vhadd<mode>"
1813   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1814         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1815                        (match_operand:VDQIW 2 "s_register_operand" "w")
1816                        (match_operand:SI 3 "immediate_operand" "i")]
1817                       UNSPEC_VHADD))]
1818   "TARGET_NEON"
1819   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1820   [(set_attr "neon_type" "neon_int_4")]
1821 )
1822
1823 (define_insn "neon_vqadd<mode>"
1824   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1825         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1826                        (match_operand:VDQIX 2 "s_register_operand" "w")
1827                        (match_operand:SI 3 "immediate_operand" "i")]
1828                      UNSPEC_VQADD))]
1829   "TARGET_NEON"
1830   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1831   [(set_attr "neon_type" "neon_int_4")]
1832 )
1833
1834 (define_insn "neon_vaddhn<mode>"
1835   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1836         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1837                             (match_operand:VN 2 "s_register_operand" "w")
1838                             (match_operand:SI 3 "immediate_operand" "i")]
1839                            UNSPEC_VADDHN))]
1840   "TARGET_NEON"
1841   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1842   [(set_attr "neon_type" "neon_int_4")]
1843 )
1844
1845 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1846 ;; polynomial multiplication case that can specified by operand 3.
1847 (define_insn "neon_vmul<mode>"
1848   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1849         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1850                       (match_operand:VDQW 2 "s_register_operand" "w")
1851                       (match_operand:SI 3 "immediate_operand" "i")]
1852                      UNSPEC_VMUL))]
1853   "TARGET_NEON"
1854   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1855   [(set (attr "neon_type")
1856       (if_then_else (match_test "<Is_float_mode>")
1857                     (if_then_else (match_test "<Is_d_reg>")
1858                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1859                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1860                     (if_then_else (match_test "<Is_d_reg>")
1861                                   (if_then_else
1862                                     (match_test "<Scalar_mul_8_16>")
1863                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1864                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1865                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1866                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
1867                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1868 )
1869
1870 (define_expand "neon_vmla<mode>"
1871   [(match_operand:VDQW 0 "s_register_operand" "=w")
1872    (match_operand:VDQW 1 "s_register_operand" "0")
1873    (match_operand:VDQW 2 "s_register_operand" "w")
1874    (match_operand:VDQW 3 "s_register_operand" "w")
1875    (match_operand:SI 4 "immediate_operand" "i")]
1876   "TARGET_NEON"
1877 {
1878   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1879     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1880                                              operands[2], operands[3]));
1881   else
1882     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1883                                            operands[2], operands[3]));
1884   DONE;
1885 })
1886
1887 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1888
1889 (define_insn "neon_vmla<mode>_unspec"
1890   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1891         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1892                      (match_operand:VDQ 2 "s_register_operand" "w")
1893                      (match_operand:VDQ 3 "s_register_operand" "w")]
1894                     UNSPEC_VMLA))]
1895   "TARGET_NEON"
1896   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1897   [(set (attr "neon_type")
1898       (if_then_else (match_test "<Is_float_mode>")
1899                     (if_then_else (match_test "<Is_d_reg>")
1900                                   (const_string "neon_fp_vmla_ddd")
1901                                   (const_string "neon_fp_vmla_qqq"))
1902                     (if_then_else (match_test "<Is_d_reg>")
1903                                   (if_then_else
1904                                     (match_test "<Scalar_mul_8_16>")
1905                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1906                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1907                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1908                                     (const_string "neon_mla_qqq_8_16")
1909                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1910 )
1911
1912 (define_insn "neon_vmlal<mode>"
1913   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1914         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1915                            (match_operand:VW 2 "s_register_operand" "w")
1916                            (match_operand:VW 3 "s_register_operand" "w")
1917                            (match_operand:SI 4 "immediate_operand" "i")]
1918                           UNSPEC_VMLAL))]
1919   "TARGET_NEON"
1920   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1921   [(set (attr "neon_type")
1922      (if_then_else (match_test "<Scalar_mul_8_16>")
1923                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1924                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1925 )
1926
1927 (define_expand "neon_vmls<mode>"
1928   [(match_operand:VDQW 0 "s_register_operand" "=w")
1929    (match_operand:VDQW 1 "s_register_operand" "0")
1930    (match_operand:VDQW 2 "s_register_operand" "w")
1931    (match_operand:VDQW 3 "s_register_operand" "w")
1932    (match_operand:SI 4 "immediate_operand" "i")]
1933   "TARGET_NEON"
1934 {
1935   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1936     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1937                  operands[1], operands[2], operands[3]));
1938   else
1939     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1940                                            operands[2], operands[3]));
1941   DONE;
1942 })
1943
1944 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1945
1946 (define_insn "neon_vmls<mode>_unspec"
1947   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1948         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1949                      (match_operand:VDQ 2 "s_register_operand" "w")
1950                      (match_operand:VDQ 3 "s_register_operand" "w")]
1951                     UNSPEC_VMLS))]
1952   "TARGET_NEON"
1953   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1954   [(set (attr "neon_type")
1955       (if_then_else (match_test "<Is_float_mode>")
1956                     (if_then_else (match_test "<Is_d_reg>")
1957                                   (const_string "neon_fp_vmla_ddd")
1958                                   (const_string "neon_fp_vmla_qqq"))
1959                     (if_then_else (match_test "<Is_d_reg>")
1960                                   (if_then_else
1961                                     (match_test "<Scalar_mul_8_16>")
1962                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1963                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1964                                   (if_then_else
1965                                     (match_test "<Scalar_mul_8_16>")
1966                                     (const_string "neon_mla_qqq_8_16")
1967                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1968 )
1969
1970 (define_insn "neon_vmlsl<mode>"
1971   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1972         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1973                            (match_operand:VW 2 "s_register_operand" "w")
1974                            (match_operand:VW 3 "s_register_operand" "w")
1975                            (match_operand:SI 4 "immediate_operand" "i")]
1976                           UNSPEC_VMLSL))]
1977   "TARGET_NEON"
1978   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1979   [(set (attr "neon_type")
1980      (if_then_else (match_test "<Scalar_mul_8_16>")
1981                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1982                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1983 )
1984
1985 (define_insn "neon_vqdmulh<mode>"
1986   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1987         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1988                        (match_operand:VMDQI 2 "s_register_operand" "w")
1989                        (match_operand:SI 3 "immediate_operand" "i")]
1990                       UNSPEC_VQDMULH))]
1991   "TARGET_NEON"
1992   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1993   [(set (attr "neon_type")
1994      (if_then_else (match_test "<Is_d_reg>")
1995         (if_then_else (match_test "<Scalar_mul_8_16>")
1996                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1997                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1998         (if_then_else (match_test "<Scalar_mul_8_16>")
1999                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2000                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2001 )
2002
2003 (define_insn "neon_vqdmlal<mode>"
2004   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2005         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2006                            (match_operand:VMDI 2 "s_register_operand" "w")
2007                            (match_operand:VMDI 3 "s_register_operand" "w")
2008                            (match_operand:SI 4 "immediate_operand" "i")]
2009                           UNSPEC_VQDMLAL))]
2010   "TARGET_NEON"
2011   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2012   [(set (attr "neon_type")
2013      (if_then_else (match_test "<Scalar_mul_8_16>")
2014                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2015                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2016 )
2017
2018 (define_insn "neon_vqdmlsl<mode>"
2019   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2020         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2021                            (match_operand:VMDI 2 "s_register_operand" "w")
2022                            (match_operand:VMDI 3 "s_register_operand" "w")
2023                            (match_operand:SI 4 "immediate_operand" "i")]
2024                           UNSPEC_VQDMLSL))]
2025   "TARGET_NEON"
2026   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2027   [(set (attr "neon_type")
2028      (if_then_else (match_test "<Scalar_mul_8_16>")
2029                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2030                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2031 )
2032
2033 (define_insn "neon_vmull<mode>"
2034   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2035         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2036                            (match_operand:VW 2 "s_register_operand" "w")
2037                            (match_operand:SI 3 "immediate_operand" "i")]
2038                           UNSPEC_VMULL))]
2039   "TARGET_NEON"
2040   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2041   [(set (attr "neon_type")
2042      (if_then_else (match_test "<Scalar_mul_8_16>")
2043                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2044                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2045 )
2046
2047 (define_insn "neon_vqdmull<mode>"
2048   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2049         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2050                            (match_operand:VMDI 2 "s_register_operand" "w")
2051                            (match_operand:SI 3 "immediate_operand" "i")]
2052                           UNSPEC_VQDMULL))]
2053   "TARGET_NEON"
2054   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2055   [(set (attr "neon_type")
2056      (if_then_else (match_test "<Scalar_mul_8_16>")
2057                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2058                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2059 )
2060
2061 (define_expand "neon_vsub<mode>"
2062   [(match_operand:VDQX 0 "s_register_operand" "=w")
2063    (match_operand:VDQX 1 "s_register_operand" "w")
2064    (match_operand:VDQX 2 "s_register_operand" "w")
2065    (match_operand:SI 3 "immediate_operand" "i")]
2066   "TARGET_NEON"
2067 {
2068   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2069     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2070   else
2071     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2072                                            operands[2]));
2073   DONE;
2074 })
2075
2076 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2077
2078 (define_insn "neon_vsub<mode>_unspec"
2079   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2080         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2081                       (match_operand:VDQX 2 "s_register_operand" "w")]
2082                      UNSPEC_VSUB))]
2083   "TARGET_NEON"
2084   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2085   [(set (attr "neon_type")
2086       (if_then_else (match_test "<Is_float_mode>")
2087                     (if_then_else (match_test "<Is_d_reg>")
2088                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2089                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2090                     (const_string "neon_int_2")))]
2091 )
2092
2093 (define_insn "neon_vsubl<mode>"
2094   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2095         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2096                            (match_operand:VDI 2 "s_register_operand" "w")
2097                            (match_operand:SI 3 "immediate_operand" "i")]
2098                           UNSPEC_VSUBL))]
2099   "TARGET_NEON"
2100   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2101   [(set_attr "neon_type" "neon_int_2")]
2102 )
2103
2104 (define_insn "neon_vsubw<mode>"
2105   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2106         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2107                            (match_operand:VDI 2 "s_register_operand" "w")
2108                            (match_operand:SI 3 "immediate_operand" "i")]
2109                           UNSPEC_VSUBW))]
2110   "TARGET_NEON"
2111   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2112   [(set_attr "neon_type" "neon_int_2")]
2113 )
2114
2115 (define_insn "neon_vqsub<mode>"
2116   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2117         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2118                        (match_operand:VDQIX 2 "s_register_operand" "w")
2119                        (match_operand:SI 3 "immediate_operand" "i")]
2120                       UNSPEC_VQSUB))]
2121   "TARGET_NEON"
2122   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2123   [(set_attr "neon_type" "neon_int_5")]
2124 )
2125
2126 (define_insn "neon_vhsub<mode>"
2127   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2128         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2129                        (match_operand:VDQIW 2 "s_register_operand" "w")
2130                        (match_operand:SI 3 "immediate_operand" "i")]
2131                       UNSPEC_VHSUB))]
2132   "TARGET_NEON"
2133   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2134   [(set_attr "neon_type" "neon_int_5")]
2135 )
2136
2137 (define_insn "neon_vsubhn<mode>"
2138   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2139         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2140                             (match_operand:VN 2 "s_register_operand" "w")
2141                             (match_operand:SI 3 "immediate_operand" "i")]
2142                            UNSPEC_VSUBHN))]
2143   "TARGET_NEON"
2144   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2145   [(set_attr "neon_type" "neon_int_4")]
2146 )
2147
2148 (define_insn "neon_vceq<mode>"
2149   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2150         (unspec:<V_cmp_result>
2151           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2152            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2153            (match_operand:SI 3 "immediate_operand" "i,i")]
2154           UNSPEC_VCEQ))]
2155   "TARGET_NEON"
2156   "@
2157   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2158   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2159   [(set (attr "neon_type")
2160       (if_then_else (match_test "<Is_float_mode>")
2161                     (if_then_else (match_test "<Is_d_reg>")
2162                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2163                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2164                     (const_string "neon_int_5")))]
2165 )
2166
2167 (define_insn "neon_vcge<mode>"
2168   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2169         (unspec:<V_cmp_result>
2170           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2171            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2172            (match_operand:SI 3 "immediate_operand" "i,i")]
2173           UNSPEC_VCGE))]
2174   "TARGET_NEON"
2175   "@
2176   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2177   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2178   [(set (attr "neon_type")
2179      (if_then_else (match_test "<Is_float_mode>")
2180                    (if_then_else (match_test "<Is_d_reg>")
2181                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2182                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2183                    (const_string "neon_int_5")))]
2184 )
2185
2186 (define_insn "neon_vcgeu<mode>"
2187   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2188         (unspec:<V_cmp_result>
2189           [(match_operand:VDQIW 1 "s_register_operand" "w")
2190            (match_operand:VDQIW 2 "s_register_operand" "w")
2191            (match_operand:SI 3 "immediate_operand" "i")]
2192           UNSPEC_VCGEU))]
2193   "TARGET_NEON"
2194   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2195   [(set_attr "neon_type" "neon_int_5")]
2196 )
2197
2198 (define_insn "neon_vcgt<mode>"
2199   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2200         (unspec:<V_cmp_result>
2201           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2202            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2203            (match_operand:SI 3 "immediate_operand" "i,i")]
2204           UNSPEC_VCGT))]
2205   "TARGET_NEON"
2206   "@
2207   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2208   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2209   [(set (attr "neon_type")
2210      (if_then_else (match_test "<Is_float_mode>")
2211                    (if_then_else (match_test "<Is_d_reg>")
2212                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2213                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2214                    (const_string "neon_int_5")))]
2215 )
2216
2217 (define_insn "neon_vcgtu<mode>"
2218   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2219         (unspec:<V_cmp_result>
2220           [(match_operand:VDQIW 1 "s_register_operand" "w")
2221            (match_operand:VDQIW 2 "s_register_operand" "w")
2222            (match_operand:SI 3 "immediate_operand" "i")]
2223           UNSPEC_VCGTU))]
2224   "TARGET_NEON"
2225   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2226   [(set_attr "neon_type" "neon_int_5")]
2227 )
2228
2229 ;; VCLE and VCLT only support comparisons with immediate zero (register
2230 ;; variants are VCGE and VCGT with operands reversed).
2231
2232 (define_insn "neon_vcle<mode>"
2233   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2234         (unspec:<V_cmp_result>
2235           [(match_operand:VDQW 1 "s_register_operand" "w")
2236            (match_operand:VDQW 2 "zero_operand" "Dz")
2237            (match_operand:SI 3 "immediate_operand" "i")]
2238           UNSPEC_VCLE))]
2239   "TARGET_NEON"
2240   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2241   [(set (attr "neon_type")
2242       (if_then_else (match_test "<Is_float_mode>")
2243                     (if_then_else (match_test "<Is_d_reg>")
2244                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2245                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2246                     (const_string "neon_int_5")))]
2247 )
2248
2249 (define_insn "neon_vclt<mode>"
2250   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2251         (unspec:<V_cmp_result>
2252           [(match_operand:VDQW 1 "s_register_operand" "w")
2253            (match_operand:VDQW 2 "zero_operand" "Dz")
2254            (match_operand:SI 3 "immediate_operand" "i")]
2255           UNSPEC_VCLT))]
2256   "TARGET_NEON"
2257   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2258   [(set (attr "neon_type")
2259       (if_then_else (match_test "<Is_float_mode>")
2260                     (if_then_else (match_test "<Is_d_reg>")
2261                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2262                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2263                     (const_string "neon_int_5")))]
2264 )
2265
2266 (define_insn "neon_vcage<mode>"
2267   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2268         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2269                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2270                                 (match_operand:SI 3 "immediate_operand" "i")]
2271                                UNSPEC_VCAGE))]
2272   "TARGET_NEON"
2273   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2274   [(set (attr "neon_type")
2275      (if_then_else (match_test "<Is_d_reg>")
2276                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2277                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2278 )
2279
2280 (define_insn "neon_vcagt<mode>"
2281   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2282         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2283                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2284                                 (match_operand:SI 3 "immediate_operand" "i")]
2285                                UNSPEC_VCAGT))]
2286   "TARGET_NEON"
2287   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2288   [(set (attr "neon_type")
2289      (if_then_else (match_test "<Is_d_reg>")
2290                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2291                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2292 )
2293
2294 (define_insn "neon_vtst<mode>"
2295   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2296         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2297                        (match_operand:VDQIW 2 "s_register_operand" "w")
2298                        (match_operand:SI 3 "immediate_operand" "i")]
2299                       UNSPEC_VTST))]
2300   "TARGET_NEON"
2301   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2302   [(set_attr "neon_type" "neon_int_4")]
2303 )
2304
2305 (define_insn "neon_vabd<mode>"
2306   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2307         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2308                       (match_operand:VDQW 2 "s_register_operand" "w")
2309                       (match_operand:SI 3 "immediate_operand" "i")]
2310                      UNSPEC_VABD))]
2311   "TARGET_NEON"
2312   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2313   [(set (attr "neon_type")
2314      (if_then_else (match_test "<Is_float_mode>")
2315                    (if_then_else (match_test "<Is_d_reg>")
2316                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2317                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2318                    (const_string "neon_int_5")))]
2319 )
2320
2321 (define_insn "neon_vabdl<mode>"
2322   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2323         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2324                            (match_operand:VW 2 "s_register_operand" "w")
2325                            (match_operand:SI 3 "immediate_operand" "i")]
2326                           UNSPEC_VABDL))]
2327   "TARGET_NEON"
2328   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2329   [(set_attr "neon_type" "neon_int_5")]
2330 )
2331
2332 (define_insn "neon_vaba<mode>"
2333   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2334         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2335                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2336                                    (match_operand:SI 4 "immediate_operand" "i")]
2337                                   UNSPEC_VABD)
2338                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2339   "TARGET_NEON"
2340   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2341   [(set (attr "neon_type")
2342      (if_then_else (match_test "<Is_d_reg>")
2343                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2344 )
2345
2346 (define_insn "neon_vabal<mode>"
2347   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2348         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2349                                            (match_operand:VW 3 "s_register_operand" "w")
2350                                            (match_operand:SI 4 "immediate_operand" "i")]
2351                                            UNSPEC_VABDL)
2352                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2353   "TARGET_NEON"
2354   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2355   [(set_attr "neon_type" "neon_vaba")]
2356 )
2357
2358 (define_insn "neon_vmax<mode>"
2359   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2360         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2361                       (match_operand:VDQW 2 "s_register_operand" "w")
2362                       (match_operand:SI 3 "immediate_operand" "i")]
2363                      UNSPEC_VMAX))]
2364   "TARGET_NEON"
2365   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2366   [(set (attr "neon_type")
2367     (if_then_else (match_test "<Is_float_mode>")
2368                   (if_then_else (match_test "<Is_d_reg>")
2369                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2370                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2371                   (const_string "neon_int_5")))]
2372 )
2373
2374 (define_insn "neon_vmin<mode>"
2375   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2376         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2377                       (match_operand:VDQW 2 "s_register_operand" "w")
2378                       (match_operand:SI 3 "immediate_operand" "i")]
2379                      UNSPEC_VMIN))]
2380   "TARGET_NEON"
2381   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2382   [(set (attr "neon_type")
2383     (if_then_else (match_test "<Is_float_mode>")
2384                   (if_then_else (match_test "<Is_d_reg>")
2385                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2386                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2387                   (const_string "neon_int_5")))]
2388 )
2389
2390 (define_expand "neon_vpadd<mode>"
2391   [(match_operand:VD 0 "s_register_operand" "=w")
2392    (match_operand:VD 1 "s_register_operand" "w")
2393    (match_operand:VD 2 "s_register_operand" "w")
2394    (match_operand:SI 3 "immediate_operand" "i")]
2395   "TARGET_NEON"
2396 {
2397   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2398                                             operands[2]));
2399   DONE;
2400 })
2401
2402 (define_insn "neon_vpaddl<mode>"
2403   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2404         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2405                                   (match_operand:SI 2 "immediate_operand" "i")]
2406                                  UNSPEC_VPADDL))]
2407   "TARGET_NEON"
2408   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2409   ;; Assume this schedules like vaddl.
2410   [(set_attr "neon_type" "neon_int_3")]
2411 )
2412
2413 (define_insn "neon_vpadal<mode>"
2414   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2415         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2416                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2417                                   (match_operand:SI 3 "immediate_operand" "i")]
2418                                  UNSPEC_VPADAL))]
2419   "TARGET_NEON"
2420   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2421   ;; Assume this schedules like vpadd.
2422   [(set_attr "neon_type" "neon_int_1")]
2423 )
2424
2425 (define_insn "neon_vpmax<mode>"
2426   [(set (match_operand:VD 0 "s_register_operand" "=w")
2427         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2428                     (match_operand:VD 2 "s_register_operand" "w")
2429                     (match_operand:SI 3 "immediate_operand" "i")]
2430                    UNSPEC_VPMAX))]
2431   "TARGET_NEON"
2432   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2433   ;; Assume this schedules like vmax.
2434   [(set (attr "neon_type")
2435     (if_then_else (match_test "<Is_float_mode>")
2436                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2437                   (const_string "neon_int_5")))]
2438 )
2439
2440 (define_insn "neon_vpmin<mode>"
2441   [(set (match_operand:VD 0 "s_register_operand" "=w")
2442         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2443                     (match_operand:VD 2 "s_register_operand" "w")
2444                     (match_operand:SI 3 "immediate_operand" "i")]
2445                    UNSPEC_VPMIN))]
2446   "TARGET_NEON"
2447   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2448   ;; Assume this schedules like vmin.
2449   [(set (attr "neon_type")
2450     (if_then_else (match_test "<Is_float_mode>")
2451                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2452                   (const_string "neon_int_5")))]
2453 )
2454
2455 (define_insn "neon_vrecps<mode>"
2456   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2457         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2458                        (match_operand:VCVTF 2 "s_register_operand" "w")
2459                        (match_operand:SI 3 "immediate_operand" "i")]
2460                       UNSPEC_VRECPS))]
2461   "TARGET_NEON"
2462   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2463   [(set (attr "neon_type")
2464       (if_then_else (match_test "<Is_d_reg>")
2465                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2466                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2467 )
2468
2469 (define_insn "neon_vrsqrts<mode>"
2470   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2471         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2472                        (match_operand:VCVTF 2 "s_register_operand" "w")
2473                        (match_operand:SI 3 "immediate_operand" "i")]
2474                       UNSPEC_VRSQRTS))]
2475   "TARGET_NEON"
2476   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2477   [(set (attr "neon_type")
2478       (if_then_else (match_test "<Is_d_reg>")
2479                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2480                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2481 )
2482
2483 (define_expand "neon_vabs<mode>"
2484   [(match_operand:VDQW 0 "s_register_operand" "")
2485    (match_operand:VDQW 1 "s_register_operand" "")
2486    (match_operand:SI 2 "immediate_operand" "")]
2487   "TARGET_NEON"
2488 {
2489   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2490   DONE;
2491 })
2492
2493 (define_insn "neon_vqabs<mode>"
2494   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2495         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2496                        (match_operand:SI 2 "immediate_operand" "i")]
2497                       UNSPEC_VQABS))]
2498   "TARGET_NEON"
2499   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2500   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2501 )
2502
2503 (define_expand "neon_vneg<mode>"
2504   [(match_operand:VDQW 0 "s_register_operand" "")
2505    (match_operand:VDQW 1 "s_register_operand" "")
2506    (match_operand:SI 2 "immediate_operand" "")]
2507   "TARGET_NEON"
2508 {
2509   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2510   DONE;
2511 })
2512
2513 (define_insn "neon_vqneg<mode>"
2514   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2515         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2516                        (match_operand:SI 2 "immediate_operand" "i")]
2517                       UNSPEC_VQNEG))]
2518   "TARGET_NEON"
2519   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2520   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2521 )
2522
2523 (define_insn "neon_vcls<mode>"
2524   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2525         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2526                        (match_operand:SI 2 "immediate_operand" "i")]
2527                       UNSPEC_VCLS))]
2528   "TARGET_NEON"
2529   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2530   [(set_attr "neon_type" "neon_int_1")]
2531 )
2532
2533 (define_insn "clz<mode>2"
2534   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2535         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2536   "TARGET_NEON"
2537   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2538   [(set_attr "neon_type" "neon_int_1")]
2539 )
2540
2541 (define_expand "neon_vclz<mode>"
2542   [(match_operand:VDQIW 0 "s_register_operand" "")
2543    (match_operand:VDQIW 1 "s_register_operand" "")
2544    (match_operand:SI 2 "immediate_operand" "")]
2545   "TARGET_NEON"
2546 {
2547   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2548   DONE;
2549 })
2550
2551 (define_insn "popcount<mode>2"
2552   [(set (match_operand:VE 0 "s_register_operand" "=w")
2553         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2554   "TARGET_NEON"
2555   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2556   [(set_attr "neon_type" "neon_int_1")]
2557 )
2558
2559 (define_expand "neon_vcnt<mode>"
2560   [(match_operand:VE 0 "s_register_operand" "=w")
2561    (match_operand:VE 1 "s_register_operand" "w")
2562    (match_operand:SI 2 "immediate_operand" "i")]
2563   "TARGET_NEON"
2564 {
2565   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2566   DONE;
2567 })
2568
2569 (define_insn "neon_vrecpe<mode>"
2570   [(set (match_operand:V32 0 "s_register_operand" "=w")
2571         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2572                      (match_operand:SI 2 "immediate_operand" "i")]
2573                     UNSPEC_VRECPE))]
2574   "TARGET_NEON"
2575   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2576   [(set (attr "neon_type")
2577       (if_then_else (match_test "<Is_d_reg>")
2578                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2579                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2580 )
2581
2582 (define_insn "neon_vrsqrte<mode>"
2583   [(set (match_operand:V32 0 "s_register_operand" "=w")
2584         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2585                      (match_operand:SI 2 "immediate_operand" "i")]
2586                     UNSPEC_VRSQRTE))]
2587   "TARGET_NEON"
2588   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2589   [(set (attr "neon_type")
2590       (if_then_else (match_test "<Is_d_reg>")
2591                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2592                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2593 )
2594
2595 (define_expand "neon_vmvn<mode>"
2596   [(match_operand:VDQIW 0 "s_register_operand" "")
2597    (match_operand:VDQIW 1 "s_register_operand" "")
2598    (match_operand:SI 2 "immediate_operand" "")]
2599   "TARGET_NEON"
2600 {
2601   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2602   DONE;
2603 })
2604
2605 (define_insn "neon_vget_lane<mode>_sext_internal"
2606   [(set (match_operand:SI 0 "s_register_operand" "=r")
2607         (sign_extend:SI
2608           (vec_select:<V_elem>
2609             (match_operand:VD 1 "s_register_operand" "w")
2610             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2611   "TARGET_NEON"
2612 {
2613   if (BYTES_BIG_ENDIAN)
2614     {
2615       int elt = INTVAL (operands[2]);
2616       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2617       operands[2] = GEN_INT (elt);
2618     }
2619   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2620 }
2621   [(set_attr "neon_type" "neon_bp_simple")]
2622 )
2623
2624 (define_insn "neon_vget_lane<mode>_zext_internal"
2625   [(set (match_operand:SI 0 "s_register_operand" "=r")
2626         (zero_extend:SI
2627           (vec_select:<V_elem>
2628             (match_operand:VD 1 "s_register_operand" "w")
2629             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2630   "TARGET_NEON"
2631 {
2632   if (BYTES_BIG_ENDIAN)
2633     {
2634       int elt = INTVAL (operands[2]);
2635       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2636       operands[2] = GEN_INT (elt);
2637     }
2638   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2639 }
2640   [(set_attr "neon_type" "neon_bp_simple")]
2641 )
2642
2643 (define_insn "neon_vget_lane<mode>_sext_internal"
2644   [(set (match_operand:SI 0 "s_register_operand" "=r")
2645         (sign_extend:SI
2646           (vec_select:<V_elem>
2647             (match_operand:VQ 1 "s_register_operand" "w")
2648             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2649   "TARGET_NEON"
2650 {
2651   rtx ops[3];
2652   int regno = REGNO (operands[1]);
2653   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2654   unsigned int elt = INTVAL (operands[2]);
2655   unsigned int elt_adj = elt % halfelts;
2656
2657   if (BYTES_BIG_ENDIAN)
2658     elt_adj = halfelts - 1 - elt_adj;
2659
2660   ops[0] = operands[0];
2661   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2662   ops[2] = GEN_INT (elt_adj);
2663   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2664
2665   return "";
2666 }
2667   [(set_attr "neon_type" "neon_bp_simple")]
2668 )
2669
2670 (define_insn "neon_vget_lane<mode>_zext_internal"
2671   [(set (match_operand:SI 0 "s_register_operand" "=r")
2672         (zero_extend:SI
2673           (vec_select:<V_elem>
2674             (match_operand:VQ 1 "s_register_operand" "w")
2675             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2676   "TARGET_NEON"
2677 {
2678   rtx ops[3];
2679   int regno = REGNO (operands[1]);
2680   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2681   unsigned int elt = INTVAL (operands[2]);
2682   unsigned int elt_adj = elt % halfelts;
2683
2684   if (BYTES_BIG_ENDIAN)
2685     elt_adj = halfelts - 1 - elt_adj;
2686
2687   ops[0] = operands[0];
2688   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2689   ops[2] = GEN_INT (elt_adj);
2690   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2691
2692   return "";
2693 }
2694   [(set_attr "neon_type" "neon_bp_simple")]
2695 )
2696
2697 (define_expand "neon_vget_lane<mode>"
2698   [(match_operand:<V_ext> 0 "s_register_operand" "")
2699    (match_operand:VDQW 1 "s_register_operand" "")
2700    (match_operand:SI 2 "immediate_operand" "")
2701    (match_operand:SI 3 "immediate_operand" "")]
2702   "TARGET_NEON"
2703 {
2704   HOST_WIDE_INT magic = INTVAL (operands[3]);
2705   rtx insn;
2706
2707   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2708
2709   if (BYTES_BIG_ENDIAN)
2710     {
2711       /* The intrinsics are defined in terms of a model where the
2712          element ordering in memory is vldm order, whereas the generic
2713          RTL is defined in terms of a model where the element ordering
2714          in memory is array order.  Convert the lane number to conform
2715          to this model.  */
2716       unsigned int elt = INTVAL (operands[2]);
2717       unsigned int reg_nelts
2718         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2719       elt ^= reg_nelts - 1;
2720       operands[2] = GEN_INT (elt);
2721     }
2722
2723   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2724     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2725   else
2726     {
2727       if ((magic & 1) != 0)
2728         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2729                                                        operands[2]);
2730       else
2731         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2732                                                        operands[2]);
2733     }
2734   emit_insn (insn);
2735   DONE;
2736 })
2737
2738 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2739 ; elements.
2740
2741 (define_expand "neon_vget_lanedi"
2742   [(match_operand:DI 0 "s_register_operand" "=r")
2743    (match_operand:DI 1 "s_register_operand" "w")
2744    (match_operand:SI 2 "immediate_operand" "i")
2745    (match_operand:SI 3 "immediate_operand" "i")]
2746   "TARGET_NEON"
2747 {
2748   neon_lane_bounds (operands[2], 0, 1);
2749   emit_move_insn (operands[0], operands[1]);
2750   DONE;
2751 })
2752
2753 (define_expand "neon_vget_lanev2di"
2754   [(match_operand:DI 0 "s_register_operand" "")
2755    (match_operand:V2DI 1 "s_register_operand" "")
2756    (match_operand:SI 2 "immediate_operand" "")
2757    (match_operand:SI 3 "immediate_operand" "")]
2758   "TARGET_NEON"
2759 {
2760   switch (INTVAL (operands[2]))
2761     {
2762     case 0:
2763       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2764       break;
2765     case 1:
2766       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2767       break;
2768     default:
2769       neon_lane_bounds (operands[2], 0, 1);
2770       FAIL;
2771     }
2772   DONE;
2773 })
2774
2775 (define_expand "neon_vset_lane<mode>"
2776   [(match_operand:VDQ 0 "s_register_operand" "=w")
2777    (match_operand:<V_elem> 1 "s_register_operand" "r")
2778    (match_operand:VDQ 2 "s_register_operand" "0")
2779    (match_operand:SI 3 "immediate_operand" "i")]
2780   "TARGET_NEON"
2781 {
2782   unsigned int elt = INTVAL (operands[3]);
2783   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2784
2785   if (BYTES_BIG_ENDIAN)
2786     {
2787       unsigned int reg_nelts
2788         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2789       elt ^= reg_nelts - 1;
2790     }
2791
2792   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2793                                          GEN_INT (1 << elt), operands[2]));
2794   DONE;
2795 })
2796
2797 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2798
2799 (define_expand "neon_vset_lanedi"
2800   [(match_operand:DI 0 "s_register_operand" "=w")
2801    (match_operand:DI 1 "s_register_operand" "r")
2802    (match_operand:DI 2 "s_register_operand" "0")
2803    (match_operand:SI 3 "immediate_operand" "i")]
2804   "TARGET_NEON"
2805 {
2806   neon_lane_bounds (operands[3], 0, 1);
2807   emit_move_insn (operands[0], operands[1]);
2808   DONE;
2809 })
2810
2811 (define_expand "neon_vcreate<mode>"
2812   [(match_operand:VDX 0 "s_register_operand" "")
2813    (match_operand:DI 1 "general_operand" "")]
2814   "TARGET_NEON"
2815 {
2816   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2817   emit_move_insn (operands[0], src);
2818   DONE;
2819 })
2820
2821 (define_insn "neon_vdup_n<mode>"
2822   [(set (match_operand:VX 0 "s_register_operand" "=w")
2823         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2824   "TARGET_NEON"
2825   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2826   ;; Assume this schedules like vmov.
2827   [(set_attr "neon_type" "neon_bp_simple")]
2828 )
2829
2830 (define_insn "neon_vdup_n<mode>"
2831   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2832         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2833   "TARGET_NEON"
2834   "@
2835   vdup.<V_sz_elem>\t%<V_reg>0, %1
2836   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
2837   ;; Assume this schedules like vmov.
2838   [(set_attr "neon_type" "neon_bp_simple")]
2839 )
2840
2841 (define_expand "neon_vdup_ndi"
2842   [(match_operand:DI 0 "s_register_operand" "=w")
2843    (match_operand:DI 1 "s_register_operand" "r")]
2844   "TARGET_NEON"
2845 {
2846   emit_move_insn (operands[0], operands[1]);
2847   DONE;
2848 }
2849 )
2850
2851 (define_insn "neon_vdup_nv2di"
2852   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2853         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2854   "TARGET_NEON"
2855   "@
2856   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
2857   vmov\t%e0, %P1\;vmov\t%f0, %P1"
2858   [(set_attr "length" "8")
2859    (set_attr "neon_type" "neon_bp_simple")]
2860 )
2861
2862 (define_insn "neon_vdup_lane<mode>_internal"
2863   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2864         (vec_duplicate:VDQW
2865           (vec_select:<V_elem>
2866             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2867             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2868   "TARGET_NEON"
2869 {
2870   if (BYTES_BIG_ENDIAN)
2871     {
2872       int elt = INTVAL (operands[2]);
2873       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2874       operands[2] = GEN_INT (elt);
2875     }
2876   if (<Is_d_reg>)
2877     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2878   else
2879     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2880 }
2881   ;; Assume this schedules like vmov.
2882   [(set_attr "neon_type" "neon_bp_simple")]
2883 )
2884
2885 (define_expand "neon_vdup_lane<mode>"
2886   [(match_operand:VDQW 0 "s_register_operand" "=w")
2887    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2888    (match_operand:SI 2 "immediate_operand" "i")]
2889   "TARGET_NEON"
2890 {
2891   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2892   if (BYTES_BIG_ENDIAN)
2893     {
2894       unsigned int elt = INTVAL (operands[2]);
2895       unsigned int reg_nelts
2896         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2897       elt ^= reg_nelts - 1;
2898       operands[2] = GEN_INT (elt);
2899     }
2900     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2901                                                   operands[2]));
2902     DONE;
2903 })
2904
2905 ; Scalar index is ignored, since only zero is valid here.
2906 (define_expand "neon_vdup_lanedi"
2907   [(match_operand:DI 0 "s_register_operand" "=w")
2908    (match_operand:DI 1 "s_register_operand" "w")
2909    (match_operand:SI 2 "immediate_operand" "i")]
2910   "TARGET_NEON"
2911 {
2912   neon_lane_bounds (operands[2], 0, 1);
2913   emit_move_insn (operands[0], operands[1]);
2914   DONE;
2915 })
2916
2917 ; Likewise for v2di, as the DImode second operand has only a single element.
2918 (define_expand "neon_vdup_lanev2di"
2919   [(match_operand:V2DI 0 "s_register_operand" "=w")
2920    (match_operand:DI 1 "s_register_operand" "w")
2921    (match_operand:SI 2 "immediate_operand" "i")]
2922   "TARGET_NEON"
2923 {
2924   neon_lane_bounds (operands[2], 0, 1);
2925   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2926   DONE;
2927 })
2928
2929 ; Disabled before reload because we don't want combine doing something silly,
2930 ; but used by the post-reload expansion of neon_vcombine.
2931 (define_insn "*neon_vswp<mode>"
2932   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2933         (match_operand:VDQX 1 "s_register_operand" "+w"))
2934    (set (match_dup 1) (match_dup 0))]
2935   "TARGET_NEON && reload_completed"
2936   "vswp\t%<V_reg>0, %<V_reg>1"
2937   [(set (attr "neon_type")
2938         (if_then_else (match_test "<Is_d_reg>")
2939                       (const_string "neon_bp_simple")
2940                       (const_string "neon_bp_2cycle")))]
2941 )
2942
2943 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2944 ;; dest vector.
2945 ;; FIXME: A different implementation of this builtin could make it much
2946 ;; more likely that we wouldn't actually need to output anything (we could make
2947 ;; it so that the reg allocator puts things in the right places magically
2948 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2949
2950 (define_insn_and_split "neon_vcombine<mode>"
2951   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2952         (vec_concat:<V_DOUBLE>
2953           (match_operand:VDX 1 "s_register_operand" "w")
2954           (match_operand:VDX 2 "s_register_operand" "w")))]
2955   "TARGET_NEON"
2956   "#"
2957   "&& reload_completed"
2958   [(const_int 0)]
2959 {
2960   neon_split_vcombine (operands);
2961   DONE;
2962 })
2963
2964 (define_expand "neon_vget_high<mode>"
2965   [(match_operand:<V_HALF> 0 "s_register_operand")
2966    (match_operand:VQX 1 "s_register_operand")]
2967   "TARGET_NEON"
2968 {
2969   emit_move_insn (operands[0],
2970                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
2971                                        GET_MODE_SIZE (<V_HALF>mode)));
2972   DONE;
2973 })
2974
2975 (define_expand "neon_vget_low<mode>"
2976   [(match_operand:<V_HALF> 0 "s_register_operand")
2977    (match_operand:VQX 1 "s_register_operand")]
2978   "TARGET_NEON"
2979 {
2980   emit_move_insn (operands[0],
2981                   simplify_gen_subreg (<V_HALF>mode, operands[1],
2982                                        <MODE>mode, 0));
2983   DONE;
2984 })
2985
2986 (define_insn "float<mode><V_cvtto>2"
2987   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2988         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
2989   "TARGET_NEON && !flag_rounding_math"
2990   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
2991   [(set (attr "neon_type")
2992      (if_then_else (match_test "<Is_d_reg>")
2993                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2994                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2995 )
2996
2997 (define_insn "floatuns<mode><V_cvtto>2"
2998   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2999         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3000   "TARGET_NEON && !flag_rounding_math"
3001   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3002   [(set (attr "neon_type")
3003      (if_then_else (match_test "<Is_d_reg>")
3004                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3005                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3006 )
3007
3008 (define_insn "fix_trunc<mode><V_cvtto>2"
3009   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3010         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3011   "TARGET_NEON"
3012   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3013   [(set (attr "neon_type")
3014      (if_then_else (match_test "<Is_d_reg>")
3015                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3016                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3017 )
3018
3019 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3020   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3021         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3022   "TARGET_NEON"
3023   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3024   [(set (attr "neon_type")
3025      (if_then_else (match_test "<Is_d_reg>")
3026                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3027                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3028 )
3029
3030 (define_insn "neon_vcvt<mode>"
3031   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3032         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3033                            (match_operand:SI 2 "immediate_operand" "i")]
3034                           UNSPEC_VCVT))]
3035   "TARGET_NEON"
3036   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3037   [(set (attr "neon_type")
3038      (if_then_else (match_test "<Is_d_reg>")
3039                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3040                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3041 )
3042
3043 (define_insn "neon_vcvt<mode>"
3044   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3045         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3046                            (match_operand:SI 2 "immediate_operand" "i")]
3047                           UNSPEC_VCVT))]
3048   "TARGET_NEON"
3049   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3050   [(set (attr "neon_type")
3051      (if_then_else (match_test "<Is_d_reg>")
3052                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3053                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3054 )
3055
3056 (define_insn "neon_vcvt_n<mode>"
3057   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3058         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3059                            (match_operand:SI 2 "immediate_operand" "i")
3060                            (match_operand:SI 3 "immediate_operand" "i")]
3061                           UNSPEC_VCVT_N))]
3062   "TARGET_NEON"
3063 {
3064   neon_const_bounds (operands[2], 1, 33);
3065   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3066 }
3067   [(set (attr "neon_type")
3068      (if_then_else (match_test "<Is_d_reg>")
3069                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3070                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3071 )
3072
3073 (define_insn "neon_vcvt_n<mode>"
3074   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3075         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3076                            (match_operand:SI 2 "immediate_operand" "i")
3077                            (match_operand:SI 3 "immediate_operand" "i")]
3078                           UNSPEC_VCVT_N))]
3079   "TARGET_NEON"
3080 {
3081   neon_const_bounds (operands[2], 1, 33);
3082   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3083 }
3084   [(set (attr "neon_type")
3085      (if_then_else (match_test "<Is_d_reg>")
3086                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3087                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3088 )
3089
3090 (define_insn "neon_vmovn<mode>"
3091   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3092         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3093                             (match_operand:SI 2 "immediate_operand" "i")]
3094                            UNSPEC_VMOVN))]
3095   "TARGET_NEON"
3096   "vmovn.<V_if_elem>\t%P0, %q1"
3097   [(set_attr "neon_type" "neon_bp_simple")]
3098 )
3099
3100 (define_insn "neon_vqmovn<mode>"
3101   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3102         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3103                             (match_operand:SI 2 "immediate_operand" "i")]
3104                            UNSPEC_VQMOVN))]
3105   "TARGET_NEON"
3106   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3107   [(set_attr "neon_type" "neon_shift_2")]
3108 )
3109
3110 (define_insn "neon_vqmovun<mode>"
3111   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3112         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3113                             (match_operand:SI 2 "immediate_operand" "i")]
3114                            UNSPEC_VQMOVUN))]
3115   "TARGET_NEON"
3116   "vqmovun.<V_s_elem>\t%P0, %q1"
3117   [(set_attr "neon_type" "neon_shift_2")]
3118 )
3119
3120 (define_insn "neon_vmovl<mode>"
3121   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3122         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3123                            (match_operand:SI 2 "immediate_operand" "i")]
3124                           UNSPEC_VMOVL))]
3125   "TARGET_NEON"
3126   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3127   [(set_attr "neon_type" "neon_shift_1")]
3128 )
3129
3130 (define_insn "neon_vmul_lane<mode>"
3131   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3132         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3133                      (match_operand:VMD 2 "s_register_operand"
3134                                         "<scalar_mul_constraint>")
3135                      (match_operand:SI 3 "immediate_operand" "i")
3136                      (match_operand:SI 4 "immediate_operand" "i")]
3137                     UNSPEC_VMUL_LANE))]
3138   "TARGET_NEON"
3139 {
3140   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3141   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3142 }
3143   [(set (attr "neon_type")
3144      (if_then_else (match_test "<Is_float_mode>")
3145                    (const_string "neon_fp_vmul_ddd")
3146                    (if_then_else (match_test "<Scalar_mul_8_16>")
3147                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3148                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3149 )
3150
3151 (define_insn "neon_vmul_lane<mode>"
3152   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3153         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3154                      (match_operand:<V_HALF> 2 "s_register_operand"
3155                                              "<scalar_mul_constraint>")
3156                      (match_operand:SI 3 "immediate_operand" "i")
3157                      (match_operand:SI 4 "immediate_operand" "i")]
3158                     UNSPEC_VMUL_LANE))]
3159   "TARGET_NEON"
3160 {
3161   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3162   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3163 }
3164   [(set (attr "neon_type")
3165      (if_then_else (match_test "<Is_float_mode>")
3166                    (const_string "neon_fp_vmul_qqd")
3167                    (if_then_else (match_test "<Scalar_mul_8_16>")
3168                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3169                                  (const_string "neon_mul_qqd_32_scalar"))))]
3170 )
3171
3172 (define_insn "neon_vmull_lane<mode>"
3173   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3174         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3175                            (match_operand:VMDI 2 "s_register_operand"
3176                                                "<scalar_mul_constraint>")
3177                            (match_operand:SI 3 "immediate_operand" "i")
3178                            (match_operand:SI 4 "immediate_operand" "i")]
3179                           UNSPEC_VMULL_LANE))]
3180   "TARGET_NEON"
3181 {
3182   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3183   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3184 }
3185   [(set (attr "neon_type")
3186      (if_then_else (match_test "<Scalar_mul_8_16>")
3187                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3188                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3189 )
3190
3191 (define_insn "neon_vqdmull_lane<mode>"
3192   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3193         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3194                            (match_operand:VMDI 2 "s_register_operand"
3195                                                "<scalar_mul_constraint>")
3196                            (match_operand:SI 3 "immediate_operand" "i")
3197                            (match_operand:SI 4 "immediate_operand" "i")]
3198                           UNSPEC_VQDMULL_LANE))]
3199   "TARGET_NEON"
3200 {
3201   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3202   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3203 }
3204   [(set (attr "neon_type")
3205      (if_then_else (match_test "<Scalar_mul_8_16>")
3206                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3207                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3208 )
3209
3210 (define_insn "neon_vqdmulh_lane<mode>"
3211   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3212         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3213                       (match_operand:<V_HALF> 2 "s_register_operand"
3214                                               "<scalar_mul_constraint>")
3215                       (match_operand:SI 3 "immediate_operand" "i")
3216                       (match_operand:SI 4 "immediate_operand" "i")]
3217                       UNSPEC_VQDMULH_LANE))]
3218   "TARGET_NEON"
3219 {
3220   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3221   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3222 }
3223   [(set (attr "neon_type")
3224      (if_then_else (match_test "<Scalar_mul_8_16>")
3225                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3226                    (const_string "neon_mul_qqd_32_scalar")))]
3227 )
3228
3229 (define_insn "neon_vqdmulh_lane<mode>"
3230   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3231         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3232                       (match_operand:VMDI 2 "s_register_operand"
3233                                           "<scalar_mul_constraint>")
3234                       (match_operand:SI 3 "immediate_operand" "i")
3235                       (match_operand:SI 4 "immediate_operand" "i")]
3236                       UNSPEC_VQDMULH_LANE))]
3237   "TARGET_NEON"
3238 {
3239   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3240   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3241 }
3242   [(set (attr "neon_type")
3243      (if_then_else (match_test "<Scalar_mul_8_16>")
3244                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3245                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3246 )
3247
3248 (define_insn "neon_vmla_lane<mode>"
3249   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3250         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3251                      (match_operand:VMD 2 "s_register_operand" "w")
3252                      (match_operand:VMD 3 "s_register_operand"
3253                                         "<scalar_mul_constraint>")
3254                      (match_operand:SI 4 "immediate_operand" "i")
3255                      (match_operand:SI 5 "immediate_operand" "i")]
3256                      UNSPEC_VMLA_LANE))]
3257   "TARGET_NEON"
3258 {
3259   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3260   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3261 }
3262   [(set (attr "neon_type")
3263      (if_then_else (match_test "<Is_float_mode>")
3264                    (const_string "neon_fp_vmla_ddd_scalar")
3265                    (if_then_else (match_test "<Scalar_mul_8_16>")
3266                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3267                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3268 )
3269
3270 (define_insn "neon_vmla_lane<mode>"
3271   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3272         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3273                      (match_operand:VMQ 2 "s_register_operand" "w")
3274                      (match_operand:<V_HALF> 3 "s_register_operand"
3275                                              "<scalar_mul_constraint>")
3276                      (match_operand:SI 4 "immediate_operand" "i")
3277                      (match_operand:SI 5 "immediate_operand" "i")]
3278                      UNSPEC_VMLA_LANE))]
3279   "TARGET_NEON"
3280 {
3281   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3282   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3283 }
3284   [(set (attr "neon_type")
3285      (if_then_else (match_test "<Is_float_mode>")
3286                    (const_string "neon_fp_vmla_qqq_scalar")
3287                    (if_then_else (match_test "<Scalar_mul_8_16>")
3288                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3289                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3290 )
3291
3292 (define_insn "neon_vmlal_lane<mode>"
3293   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3294         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3295                            (match_operand:VMDI 2 "s_register_operand" "w")
3296                            (match_operand:VMDI 3 "s_register_operand"
3297                                                "<scalar_mul_constraint>")
3298                            (match_operand:SI 4 "immediate_operand" "i")
3299                            (match_operand:SI 5 "immediate_operand" "i")]
3300                           UNSPEC_VMLAL_LANE))]
3301   "TARGET_NEON"
3302 {
3303   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3304   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3305 }
3306   [(set (attr "neon_type")
3307      (if_then_else (match_test "<Scalar_mul_8_16>")
3308                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3309                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3310 )
3311
3312 (define_insn "neon_vqdmlal_lane<mode>"
3313   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3314         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3315                            (match_operand:VMDI 2 "s_register_operand" "w")
3316                            (match_operand:VMDI 3 "s_register_operand"
3317                                                "<scalar_mul_constraint>")
3318                            (match_operand:SI 4 "immediate_operand" "i")
3319                            (match_operand:SI 5 "immediate_operand" "i")]
3320                           UNSPEC_VQDMLAL_LANE))]
3321   "TARGET_NEON"
3322 {
3323   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3324   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3325 }
3326   [(set (attr "neon_type")
3327      (if_then_else (match_test "<Scalar_mul_8_16>")
3328                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3329                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3330 )
3331
3332 (define_insn "neon_vmls_lane<mode>"
3333   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3334         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3335                      (match_operand:VMD 2 "s_register_operand" "w")
3336                      (match_operand:VMD 3 "s_register_operand"
3337                                         "<scalar_mul_constraint>")
3338                      (match_operand:SI 4 "immediate_operand" "i")
3339                      (match_operand:SI 5 "immediate_operand" "i")]
3340                     UNSPEC_VMLS_LANE))]
3341   "TARGET_NEON"
3342 {
3343   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3344   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3345 }
3346   [(set (attr "neon_type")
3347      (if_then_else (match_test "<Is_float_mode>")
3348                    (const_string "neon_fp_vmla_ddd_scalar")
3349                    (if_then_else (match_test "<Scalar_mul_8_16>")
3350                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3351                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3352 )
3353
3354 (define_insn "neon_vmls_lane<mode>"
3355   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3356         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3357                      (match_operand:VMQ 2 "s_register_operand" "w")
3358                      (match_operand:<V_HALF> 3 "s_register_operand"
3359                                              "<scalar_mul_constraint>")
3360                      (match_operand:SI 4 "immediate_operand" "i")
3361                      (match_operand:SI 5 "immediate_operand" "i")]
3362                     UNSPEC_VMLS_LANE))]
3363   "TARGET_NEON"
3364 {
3365   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3366   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3367 }
3368   [(set (attr "neon_type")
3369      (if_then_else (match_test "<Is_float_mode>")
3370                    (const_string "neon_fp_vmla_qqq_scalar")
3371                    (if_then_else (match_test "<Scalar_mul_8_16>")
3372                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3373                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3374 )
3375
3376 (define_insn "neon_vmlsl_lane<mode>"
3377   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3378         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3379                            (match_operand:VMDI 2 "s_register_operand" "w")
3380                            (match_operand:VMDI 3 "s_register_operand"
3381                                                "<scalar_mul_constraint>")
3382                            (match_operand:SI 4 "immediate_operand" "i")
3383                            (match_operand:SI 5 "immediate_operand" "i")]
3384                           UNSPEC_VMLSL_LANE))]
3385   "TARGET_NEON"
3386 {
3387   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3388   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3389 }
3390   [(set (attr "neon_type")
3391      (if_then_else (match_test "<Scalar_mul_8_16>")
3392                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3393                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3394 )
3395
3396 (define_insn "neon_vqdmlsl_lane<mode>"
3397   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3398         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3399                            (match_operand:VMDI 2 "s_register_operand" "w")
3400                            (match_operand:VMDI 3 "s_register_operand"
3401                                                "<scalar_mul_constraint>")
3402                            (match_operand:SI 4 "immediate_operand" "i")
3403                            (match_operand:SI 5 "immediate_operand" "i")]
3404                           UNSPEC_VQDMLSL_LANE))]
3405   "TARGET_NEON"
3406 {
3407   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3408   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3409 }
3410   [(set (attr "neon_type")
3411      (if_then_else (match_test "<Scalar_mul_8_16>")
3412                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3413                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3414 )
3415
3416 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3417 ; core register into a temp register, then use a scalar taken from that. This
3418 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3419 ; or extracted from another vector. The latter case it's currently better to
3420 ; use the "_lane" variant, and the former case can probably be implemented
3421 ; using vld1_lane, but that hasn't been done yet.
3422
3423 (define_expand "neon_vmul_n<mode>"
3424   [(match_operand:VMD 0 "s_register_operand" "")
3425    (match_operand:VMD 1 "s_register_operand" "")
3426    (match_operand:<V_elem> 2 "s_register_operand" "")
3427    (match_operand:SI 3 "immediate_operand" "")]
3428   "TARGET_NEON"
3429 {
3430   rtx tmp = gen_reg_rtx (<MODE>mode);
3431   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3432   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3433                                        const0_rtx, const0_rtx));
3434   DONE;
3435 })
3436
3437 (define_expand "neon_vmul_n<mode>"
3438   [(match_operand:VMQ 0 "s_register_operand" "")
3439    (match_operand:VMQ 1 "s_register_operand" "")
3440    (match_operand:<V_elem> 2 "s_register_operand" "")
3441    (match_operand:SI 3 "immediate_operand" "")]
3442   "TARGET_NEON"
3443 {
3444   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3445   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3446   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3447                                        const0_rtx, const0_rtx));
3448   DONE;
3449 })
3450
3451 (define_expand "neon_vmull_n<mode>"
3452   [(match_operand:<V_widen> 0 "s_register_operand" "")
3453    (match_operand:VMDI 1 "s_register_operand" "")
3454    (match_operand:<V_elem> 2 "s_register_operand" "")
3455    (match_operand:SI 3 "immediate_operand" "")]
3456   "TARGET_NEON"
3457 {
3458   rtx tmp = gen_reg_rtx (<MODE>mode);
3459   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3460   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3461                                         const0_rtx, operands[3]));
3462   DONE;
3463 })
3464
3465 (define_expand "neon_vqdmull_n<mode>"
3466   [(match_operand:<V_widen> 0 "s_register_operand" "")
3467    (match_operand:VMDI 1 "s_register_operand" "")
3468    (match_operand:<V_elem> 2 "s_register_operand" "")
3469    (match_operand:SI 3 "immediate_operand" "")]
3470   "TARGET_NEON"
3471 {
3472   rtx tmp = gen_reg_rtx (<MODE>mode);
3473   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3474   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3475                                           const0_rtx, const0_rtx));
3476   DONE;
3477 })
3478
3479 (define_expand "neon_vqdmulh_n<mode>"
3480   [(match_operand:VMDI 0 "s_register_operand" "")
3481    (match_operand:VMDI 1 "s_register_operand" "")
3482    (match_operand:<V_elem> 2 "s_register_operand" "")
3483    (match_operand:SI 3 "immediate_operand" "")]
3484   "TARGET_NEON"
3485 {
3486   rtx tmp = gen_reg_rtx (<MODE>mode);
3487   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3488   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3489                                           const0_rtx, operands[3]));
3490   DONE;
3491 })
3492
3493 (define_expand "neon_vqdmulh_n<mode>"
3494   [(match_operand:VMQI 0 "s_register_operand" "")
3495    (match_operand:VMQI 1 "s_register_operand" "")
3496    (match_operand:<V_elem> 2 "s_register_operand" "")
3497    (match_operand:SI 3 "immediate_operand" "")]
3498   "TARGET_NEON"
3499 {
3500   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3501   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3502   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3503                                           const0_rtx, operands[3]));
3504   DONE;
3505 })
3506
3507 (define_expand "neon_vmla_n<mode>"
3508   [(match_operand:VMD 0 "s_register_operand" "")
3509    (match_operand:VMD 1 "s_register_operand" "")
3510    (match_operand:VMD 2 "s_register_operand" "")
3511    (match_operand:<V_elem> 3 "s_register_operand" "")
3512    (match_operand:SI 4 "immediate_operand" "")]
3513   "TARGET_NEON"
3514 {
3515   rtx tmp = gen_reg_rtx (<MODE>mode);
3516   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3517   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3518                                        tmp, const0_rtx, operands[4]));
3519   DONE;
3520 })
3521
3522 (define_expand "neon_vmla_n<mode>"
3523   [(match_operand:VMQ 0 "s_register_operand" "")
3524    (match_operand:VMQ 1 "s_register_operand" "")
3525    (match_operand:VMQ 2 "s_register_operand" "")
3526    (match_operand:<V_elem> 3 "s_register_operand" "")
3527    (match_operand:SI 4 "immediate_operand" "")]
3528   "TARGET_NEON"
3529 {
3530   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3531   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3532   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3533                                        tmp, const0_rtx, operands[4]));
3534   DONE;
3535 })
3536
3537 (define_expand "neon_vmlal_n<mode>"
3538   [(match_operand:<V_widen> 0 "s_register_operand" "")
3539    (match_operand:<V_widen> 1 "s_register_operand" "")
3540    (match_operand:VMDI 2 "s_register_operand" "")
3541    (match_operand:<V_elem> 3 "s_register_operand" "")
3542    (match_operand:SI 4 "immediate_operand" "")]
3543   "TARGET_NEON"
3544 {
3545   rtx tmp = gen_reg_rtx (<MODE>mode);
3546   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3547   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3548                                         tmp, const0_rtx, operands[4]));
3549   DONE;
3550 })
3551
3552 (define_expand "neon_vqdmlal_n<mode>"
3553   [(match_operand:<V_widen> 0 "s_register_operand" "")
3554    (match_operand:<V_widen> 1 "s_register_operand" "")
3555    (match_operand:VMDI 2 "s_register_operand" "")
3556    (match_operand:<V_elem> 3 "s_register_operand" "")
3557    (match_operand:SI 4 "immediate_operand" "")]
3558   "TARGET_NEON"
3559 {
3560   rtx tmp = gen_reg_rtx (<MODE>mode);
3561   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3562   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3563                                           tmp, const0_rtx, operands[4]));
3564   DONE;
3565 })
3566
3567 (define_expand "neon_vmls_n<mode>"
3568   [(match_operand:VMD 0 "s_register_operand" "")
3569    (match_operand:VMD 1 "s_register_operand" "")
3570    (match_operand:VMD 2 "s_register_operand" "")
3571    (match_operand:<V_elem> 3 "s_register_operand" "")
3572    (match_operand:SI 4 "immediate_operand" "")]
3573   "TARGET_NEON"
3574 {
3575   rtx tmp = gen_reg_rtx (<MODE>mode);
3576   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3577   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3578                                        tmp, const0_rtx, operands[4]));
3579   DONE;
3580 })
3581
3582 (define_expand "neon_vmls_n<mode>"
3583   [(match_operand:VMQ 0 "s_register_operand" "")
3584    (match_operand:VMQ 1 "s_register_operand" "")
3585    (match_operand:VMQ 2 "s_register_operand" "")
3586    (match_operand:<V_elem> 3 "s_register_operand" "")
3587    (match_operand:SI 4 "immediate_operand" "")]
3588   "TARGET_NEON"
3589 {
3590   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3591   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3592   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3593                                        tmp, const0_rtx, operands[4]));
3594   DONE;
3595 })
3596
3597 (define_expand "neon_vmlsl_n<mode>"
3598   [(match_operand:<V_widen> 0 "s_register_operand" "")
3599    (match_operand:<V_widen> 1 "s_register_operand" "")
3600    (match_operand:VMDI 2 "s_register_operand" "")
3601    (match_operand:<V_elem> 3 "s_register_operand" "")
3602    (match_operand:SI 4 "immediate_operand" "")]
3603   "TARGET_NEON"
3604 {
3605   rtx tmp = gen_reg_rtx (<MODE>mode);
3606   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3607   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3608                                         tmp, const0_rtx, operands[4]));
3609   DONE;
3610 })
3611
3612 (define_expand "neon_vqdmlsl_n<mode>"
3613   [(match_operand:<V_widen> 0 "s_register_operand" "")
3614    (match_operand:<V_widen> 1 "s_register_operand" "")
3615    (match_operand:VMDI 2 "s_register_operand" "")
3616    (match_operand:<V_elem> 3 "s_register_operand" "")
3617    (match_operand:SI 4 "immediate_operand" "")]
3618   "TARGET_NEON"
3619 {
3620   rtx tmp = gen_reg_rtx (<MODE>mode);
3621   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3622   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3623                                           tmp, const0_rtx, operands[4]));
3624   DONE;
3625 })
3626
3627 (define_insn "neon_vext<mode>"
3628   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3629         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3630                       (match_operand:VDQX 2 "s_register_operand" "w")
3631                       (match_operand:SI 3 "immediate_operand" "i")]
3632                      UNSPEC_VEXT))]
3633   "TARGET_NEON"
3634 {
3635   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3636   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3637 }
3638   [(set (attr "neon_type")
3639       (if_then_else (match_test "<Is_d_reg>")
3640                     (const_string "neon_bp_simple")
3641                     (const_string "neon_bp_2cycle")))]
3642 )
3643
3644 (define_insn "neon_vrev64<mode>"
3645   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3646         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3647                      (match_operand:SI 2 "immediate_operand" "i")]
3648                     UNSPEC_VREV64))]
3649   "TARGET_NEON"
3650   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3651   [(set_attr "neon_type" "neon_bp_simple")]
3652 )
3653
3654 (define_insn "neon_vrev32<mode>"
3655   [(set (match_operand:VX 0 "s_register_operand" "=w")
3656         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3657                     (match_operand:SI 2 "immediate_operand" "i")]
3658                    UNSPEC_VREV32))]
3659   "TARGET_NEON"
3660   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3661   [(set_attr "neon_type" "neon_bp_simple")]
3662 )
3663
3664 (define_insn "neon_vrev16<mode>"
3665   [(set (match_operand:VE 0 "s_register_operand" "=w")
3666         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3667                     (match_operand:SI 2 "immediate_operand" "i")]
3668                    UNSPEC_VREV16))]
3669   "TARGET_NEON"
3670   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3671   [(set_attr "neon_type" "neon_bp_simple")]
3672 )
3673
3674 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3675 ; allocation. For an intrinsic of form:
3676 ;   rD = vbsl_* (rS, rN, rM)
3677 ; We can use any of:
3678 ;   vbsl rS, rN, rM  (if D = S)
3679 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3680 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3681
3682 (define_insn "neon_vbsl<mode>_internal"
3683   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3684         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3685                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3686                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3687                      UNSPEC_VBSL))]
3688   "TARGET_NEON"
3689   "@
3690   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3691   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3692   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3693   [(set_attr "neon_type" "neon_int_1")]
3694 )
3695
3696 (define_expand "neon_vbsl<mode>"
3697   [(set (match_operand:VDQX 0 "s_register_operand" "")
3698         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3699                       (match_operand:VDQX 2 "s_register_operand" "")
3700                       (match_operand:VDQX 3 "s_register_operand" "")]
3701                      UNSPEC_VBSL))]
3702   "TARGET_NEON"
3703 {
3704   /* We can't alias operands together if they have different modes.  */
3705   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3706 })
3707
3708 (define_insn "neon_vshl<mode>"
3709   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3710         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3711                        (match_operand:VDQIX 2 "s_register_operand" "w")
3712                        (match_operand:SI 3 "immediate_operand" "i")]
3713                       UNSPEC_VSHL))]
3714   "TARGET_NEON"
3715   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3716   [(set (attr "neon_type")
3717       (if_then_else (match_test "<Is_d_reg>")
3718                     (const_string "neon_vshl_ddd")
3719                     (const_string "neon_shift_3")))]
3720 )
3721
3722 (define_insn "neon_vqshl<mode>"
3723   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3724         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3725                        (match_operand:VDQIX 2 "s_register_operand" "w")
3726                        (match_operand:SI 3 "immediate_operand" "i")]
3727                       UNSPEC_VQSHL))]
3728   "TARGET_NEON"
3729   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3730   [(set (attr "neon_type")
3731       (if_then_else (match_test "<Is_d_reg>")
3732                     (const_string "neon_shift_2")
3733                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3734 )
3735
3736 (define_insn "neon_vshr_n<mode>"
3737   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3738         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3739                        (match_operand:SI 2 "immediate_operand" "i")
3740                        (match_operand:SI 3 "immediate_operand" "i")]
3741                       UNSPEC_VSHR_N))]
3742   "TARGET_NEON"
3743 {
3744   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3745   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3746 }
3747   [(set_attr "neon_type" "neon_shift_1")]
3748 )
3749
3750 (define_insn "neon_vshrn_n<mode>"
3751   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3752         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3753                             (match_operand:SI 2 "immediate_operand" "i")
3754                             (match_operand:SI 3 "immediate_operand" "i")]
3755                            UNSPEC_VSHRN_N))]
3756   "TARGET_NEON"
3757 {
3758   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3759   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3760 }
3761   [(set_attr "neon_type" "neon_shift_1")]
3762 )
3763
3764 (define_insn "neon_vqshrn_n<mode>"
3765   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3766         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3767                             (match_operand:SI 2 "immediate_operand" "i")
3768                             (match_operand:SI 3 "immediate_operand" "i")]
3769                            UNSPEC_VQSHRN_N))]
3770   "TARGET_NEON"
3771 {
3772   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3773   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3774 }
3775   [(set_attr "neon_type" "neon_shift_2")]
3776 )
3777
3778 (define_insn "neon_vqshrun_n<mode>"
3779   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3780         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3781                             (match_operand:SI 2 "immediate_operand" "i")
3782                             (match_operand:SI 3 "immediate_operand" "i")]
3783                            UNSPEC_VQSHRUN_N))]
3784   "TARGET_NEON"
3785 {
3786   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3787   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3788 }
3789   [(set_attr "neon_type" "neon_shift_2")]
3790 )
3791
3792 (define_insn "neon_vshl_n<mode>"
3793   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3794         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3795                        (match_operand:SI 2 "immediate_operand" "i")
3796                        (match_operand:SI 3 "immediate_operand" "i")]
3797                       UNSPEC_VSHL_N))]
3798   "TARGET_NEON"
3799 {
3800   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3801   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3802 }
3803   [(set_attr "neon_type" "neon_shift_1")]
3804 )
3805
3806 (define_insn "neon_vqshl_n<mode>"
3807   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3808         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3809                        (match_operand:SI 2 "immediate_operand" "i")
3810                        (match_operand:SI 3 "immediate_operand" "i")]
3811                       UNSPEC_VQSHL_N))]
3812   "TARGET_NEON"
3813 {
3814   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3815   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3816 }
3817   [(set_attr "neon_type" "neon_shift_2")]
3818 )
3819
3820 (define_insn "neon_vqshlu_n<mode>"
3821   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3822         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3823                        (match_operand:SI 2 "immediate_operand" "i")
3824                        (match_operand:SI 3 "immediate_operand" "i")]
3825                       UNSPEC_VQSHLU_N))]
3826   "TARGET_NEON"
3827 {
3828   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3829   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3830 }
3831   [(set_attr "neon_type" "neon_shift_2")]
3832 )
3833
3834 (define_insn "neon_vshll_n<mode>"
3835   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3836         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3837                            (match_operand:SI 2 "immediate_operand" "i")
3838                            (match_operand:SI 3 "immediate_operand" "i")]
3839                           UNSPEC_VSHLL_N))]
3840   "TARGET_NEON"
3841 {
3842   /* The boundaries are: 0 < imm <= size.  */
3843   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3844   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3845 }
3846   [(set_attr "neon_type" "neon_shift_1")]
3847 )
3848
3849 (define_insn "neon_vsra_n<mode>"
3850   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3851         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3852                        (match_operand:VDQIX 2 "s_register_operand" "w")
3853                        (match_operand:SI 3 "immediate_operand" "i")
3854                        (match_operand:SI 4 "immediate_operand" "i")]
3855                       UNSPEC_VSRA_N))]
3856   "TARGET_NEON"
3857 {
3858   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3859   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3860 }
3861   [(set_attr "neon_type" "neon_vsra_vrsra")]
3862 )
3863
3864 (define_insn "neon_vsri_n<mode>"
3865   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3866         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3867                        (match_operand:VDQIX 2 "s_register_operand" "w")
3868                        (match_operand:SI 3 "immediate_operand" "i")]
3869                       UNSPEC_VSRI))]
3870   "TARGET_NEON"
3871 {
3872   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3873   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3874 }
3875   [(set (attr "neon_type")
3876       (if_then_else (match_test "<Is_d_reg>")
3877                     (const_string "neon_shift_1")
3878                     (const_string "neon_shift_3")))]
3879 )
3880
3881 (define_insn "neon_vsli_n<mode>"
3882   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3883         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3884                        (match_operand:VDQIX 2 "s_register_operand" "w")
3885                        (match_operand:SI 3 "immediate_operand" "i")]
3886                       UNSPEC_VSLI))]
3887   "TARGET_NEON"
3888 {
3889   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3890   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3891 }
3892   [(set (attr "neon_type")
3893       (if_then_else (match_test "<Is_d_reg>")
3894                     (const_string "neon_shift_1")
3895                     (const_string "neon_shift_3")))]
3896 )
3897
3898 (define_insn "neon_vtbl1v8qi"
3899   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3900         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3901                       (match_operand:V8QI 2 "s_register_operand" "w")]
3902                      UNSPEC_VTBL))]
3903   "TARGET_NEON"
3904   "vtbl.8\t%P0, {%P1}, %P2"
3905   [(set_attr "neon_type" "neon_bp_2cycle")]
3906 )
3907
3908 (define_insn "neon_vtbl2v8qi"
3909   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3910         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3911                       (match_operand:V8QI 2 "s_register_operand" "w")]
3912                      UNSPEC_VTBL))]
3913   "TARGET_NEON"
3914 {
3915   rtx ops[4];
3916   int tabbase = REGNO (operands[1]);
3917
3918   ops[0] = operands[0];
3919   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3920   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3921   ops[3] = operands[2];
3922   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3923
3924   return "";
3925 }
3926   [(set_attr "neon_type" "neon_bp_2cycle")]
3927 )
3928
3929 (define_insn "neon_vtbl3v8qi"
3930   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3931         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3932                       (match_operand:V8QI 2 "s_register_operand" "w")]
3933                      UNSPEC_VTBL))]
3934   "TARGET_NEON"
3935 {
3936   rtx ops[5];
3937   int tabbase = REGNO (operands[1]);
3938
3939   ops[0] = operands[0];
3940   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3941   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3942   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3943   ops[4] = operands[2];
3944   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3945
3946   return "";
3947 }
3948   [(set_attr "neon_type" "neon_bp_3cycle")]
3949 )
3950
3951 (define_insn "neon_vtbl4v8qi"
3952   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3953         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3954                       (match_operand:V8QI 2 "s_register_operand" "w")]
3955                      UNSPEC_VTBL))]
3956   "TARGET_NEON"
3957 {
3958   rtx ops[6];
3959   int tabbase = REGNO (operands[1]);
3960
3961   ops[0] = operands[0];
3962   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3963   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3964   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3965   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3966   ops[5] = operands[2];
3967   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3968
3969   return "";
3970 }
3971   [(set_attr "neon_type" "neon_bp_3cycle")]
3972 )
3973
3974 ;; These three are used by the vec_perm infrastructure for V16QImode.
3975 (define_insn_and_split "neon_vtbl1v16qi"
3976   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3977         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
3978                        (match_operand:V16QI 2 "s_register_operand" "w")]
3979                       UNSPEC_VTBL))]
3980   "TARGET_NEON"
3981   "#"
3982   "&& reload_completed"
3983   [(const_int 0)]
3984 {
3985   rtx op0, op1, op2, part0, part2;
3986   unsigned ofs;
3987
3988   op0 = operands[0];
3989   op1 = gen_lowpart (TImode, operands[1]);
3990   op2 = operands[2];
3991
3992   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3993   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3994   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3995   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3996
3997   ofs = subreg_highpart_offset (V8QImode, V16QImode);
3998   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3999   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4000   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4001   DONE;
4002 })
4003
4004 (define_insn_and_split "neon_vtbl2v16qi"
4005   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4006         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4007                        (match_operand:V16QI 2 "s_register_operand" "w")]
4008                       UNSPEC_VTBL))]
4009   "TARGET_NEON"
4010   "#"
4011   "&& reload_completed"
4012   [(const_int 0)]
4013 {
4014   rtx op0, op1, op2, part0, part2;
4015   unsigned ofs;
4016
4017   op0 = operands[0];
4018   op1 = operands[1];
4019   op2 = operands[2];
4020
4021   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4022   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4023   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4024   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4025
4026   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4027   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4028   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4029   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4030   DONE;
4031 })
4032
4033 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4034 ;; handle quad-word input modes, producing octa-word output modes.  But
4035 ;; that requires us to add support for octa-word vector modes in moves.
4036 ;; That seems overkill for this one use in vec_perm.
4037 (define_insn_and_split "neon_vcombinev16qi"
4038   [(set (match_operand:OI 0 "s_register_operand" "=w")
4039         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4040                     (match_operand:V16QI 2 "s_register_operand" "w")]
4041                    UNSPEC_VCONCAT))]
4042   "TARGET_NEON"
4043   "#"
4044   "&& reload_completed"
4045   [(const_int 0)]
4046 {
4047   neon_split_vcombine (operands);
4048   DONE;
4049 })
4050
4051 (define_insn "neon_vtbx1v8qi"
4052   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4053         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4054                       (match_operand:V8QI 2 "s_register_operand" "w")
4055                       (match_operand:V8QI 3 "s_register_operand" "w")]
4056                      UNSPEC_VTBX))]
4057   "TARGET_NEON"
4058   "vtbx.8\t%P0, {%P2}, %P3"
4059   [(set_attr "neon_type" "neon_bp_2cycle")]
4060 )
4061
4062 (define_insn "neon_vtbx2v8qi"
4063   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4064         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4065                       (match_operand:TI 2 "s_register_operand" "w")
4066                       (match_operand:V8QI 3 "s_register_operand" "w")]
4067                      UNSPEC_VTBX))]
4068   "TARGET_NEON"
4069 {
4070   rtx ops[4];
4071   int tabbase = REGNO (operands[2]);
4072
4073   ops[0] = operands[0];
4074   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4075   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4076   ops[3] = operands[3];
4077   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4078
4079   return "";
4080 }
4081   [(set_attr "neon_type" "neon_bp_2cycle")]
4082 )
4083
4084 (define_insn "neon_vtbx3v8qi"
4085   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4086         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4087                       (match_operand:EI 2 "s_register_operand" "w")
4088                       (match_operand:V8QI 3 "s_register_operand" "w")]
4089                      UNSPEC_VTBX))]
4090   "TARGET_NEON"
4091 {
4092   rtx ops[5];
4093   int tabbase = REGNO (operands[2]);
4094
4095   ops[0] = operands[0];
4096   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4097   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4098   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4099   ops[4] = operands[3];
4100   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4101
4102   return "";
4103 }
4104   [(set_attr "neon_type" "neon_bp_3cycle")]
4105 )
4106
4107 (define_insn "neon_vtbx4v8qi"
4108   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4109         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4110                       (match_operand:OI 2 "s_register_operand" "w")
4111                       (match_operand:V8QI 3 "s_register_operand" "w")]
4112                      UNSPEC_VTBX))]
4113   "TARGET_NEON"
4114 {
4115   rtx ops[6];
4116   int tabbase = REGNO (operands[2]);
4117
4118   ops[0] = operands[0];
4119   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4120   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4121   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4122   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4123   ops[5] = operands[3];
4124   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4125
4126   return "";
4127 }
4128   [(set_attr "neon_type" "neon_bp_3cycle")]
4129 )
4130
4131 (define_insn "neon_vtrn<mode>_internal"
4132   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4133         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4134                       (match_operand:VDQW 2 "s_register_operand" "w")]
4135                      UNSPEC_VTRN1))
4136    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4137          (unspec:VDQW [(match_dup 1) (match_dup 2)]
4138                      UNSPEC_VTRN2))]
4139   "TARGET_NEON"
4140   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4141   [(set (attr "neon_type")
4142       (if_then_else (match_test "<Is_d_reg>")
4143                     (const_string "neon_bp_simple")
4144                     (const_string "neon_bp_3cycle")))]
4145 )
4146
4147 (define_expand "neon_vtrn<mode>"
4148   [(match_operand:SI 0 "s_register_operand" "r")
4149    (match_operand:VDQW 1 "s_register_operand" "w")
4150    (match_operand:VDQW 2 "s_register_operand" "w")]
4151   "TARGET_NEON"
4152 {
4153   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4154                               operands[0], operands[1], operands[2]);
4155   DONE;
4156 })
4157
4158 (define_insn "neon_vzip<mode>_internal"
4159   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4160         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4161                       (match_operand:VDQW 2 "s_register_operand" "w")]
4162                      UNSPEC_VZIP1))
4163    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4164         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4165                      UNSPEC_VZIP2))]
4166   "TARGET_NEON"
4167   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4168   [(set (attr "neon_type")
4169       (if_then_else (match_test "<Is_d_reg>")
4170                     (const_string "neon_bp_simple")
4171                     (const_string "neon_bp_3cycle")))]
4172 )
4173
4174 (define_expand "neon_vzip<mode>"
4175   [(match_operand:SI 0 "s_register_operand" "r")
4176    (match_operand:VDQW 1 "s_register_operand" "w")
4177    (match_operand:VDQW 2 "s_register_operand" "w")]
4178   "TARGET_NEON"
4179 {
4180   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4181                               operands[0], operands[1], operands[2]);
4182   DONE;
4183 })
4184
4185 (define_insn "neon_vuzp<mode>_internal"
4186   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4187         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4188                       (match_operand:VDQW 2 "s_register_operand" "w")]
4189                      UNSPEC_VUZP1))
4190    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4191         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4192                      UNSPEC_VUZP2))]
4193   "TARGET_NEON"
4194   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4195   [(set (attr "neon_type")
4196       (if_then_else (match_test "<Is_d_reg>")
4197                     (const_string "neon_bp_simple")
4198                     (const_string "neon_bp_3cycle")))]
4199 )
4200
4201 (define_expand "neon_vuzp<mode>"
4202   [(match_operand:SI 0 "s_register_operand" "r")
4203    (match_operand:VDQW 1 "s_register_operand" "w")
4204    (match_operand:VDQW 2 "s_register_operand" "w")]
4205   "TARGET_NEON"
4206 {
4207   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4208                               operands[0], operands[1], operands[2]);
4209   DONE;
4210 })
4211
4212 (define_expand "neon_vreinterpretv8qi<mode>"
4213   [(match_operand:V8QI 0 "s_register_operand" "")
4214    (match_operand:VDX 1 "s_register_operand" "")]
4215   "TARGET_NEON"
4216 {
4217   neon_reinterpret (operands[0], operands[1]);
4218   DONE;
4219 })
4220
4221 (define_expand "neon_vreinterpretv4hi<mode>"
4222   [(match_operand:V4HI 0 "s_register_operand" "")
4223    (match_operand:VDX 1 "s_register_operand" "")]
4224   "TARGET_NEON"
4225 {
4226   neon_reinterpret (operands[0], operands[1]);
4227   DONE;
4228 })
4229
4230 (define_expand "neon_vreinterpretv2si<mode>"
4231   [(match_operand:V2SI 0 "s_register_operand" "")
4232    (match_operand:VDX 1 "s_register_operand" "")]
4233   "TARGET_NEON"
4234 {
4235   neon_reinterpret (operands[0], operands[1]);
4236   DONE;
4237 })
4238
4239 (define_expand "neon_vreinterpretv2sf<mode>"
4240   [(match_operand:V2SF 0 "s_register_operand" "")
4241    (match_operand:VDX 1 "s_register_operand" "")]
4242   "TARGET_NEON"
4243 {
4244   neon_reinterpret (operands[0], operands[1]);
4245   DONE;
4246 })
4247
4248 (define_expand "neon_vreinterpretdi<mode>"
4249   [(match_operand:DI 0 "s_register_operand" "")
4250    (match_operand:VDX 1 "s_register_operand" "")]
4251   "TARGET_NEON"
4252 {
4253   neon_reinterpret (operands[0], operands[1]);
4254   DONE;
4255 })
4256
4257 (define_expand "neon_vreinterpretv16qi<mode>"
4258   [(match_operand:V16QI 0 "s_register_operand" "")
4259    (match_operand:VQX 1 "s_register_operand" "")]
4260   "TARGET_NEON"
4261 {
4262   neon_reinterpret (operands[0], operands[1]);
4263   DONE;
4264 })
4265
4266 (define_expand "neon_vreinterpretv8hi<mode>"
4267   [(match_operand:V8HI 0 "s_register_operand" "")
4268    (match_operand:VQX 1 "s_register_operand" "")]
4269   "TARGET_NEON"
4270 {
4271   neon_reinterpret (operands[0], operands[1]);
4272   DONE;
4273 })
4274
4275 (define_expand "neon_vreinterpretv4si<mode>"
4276   [(match_operand:V4SI 0 "s_register_operand" "")
4277    (match_operand:VQX 1 "s_register_operand" "")]
4278   "TARGET_NEON"
4279 {
4280   neon_reinterpret (operands[0], operands[1]);
4281   DONE;
4282 })
4283
4284 (define_expand "neon_vreinterpretv4sf<mode>"
4285   [(match_operand:V4SF 0 "s_register_operand" "")
4286    (match_operand:VQX 1 "s_register_operand" "")]
4287   "TARGET_NEON"
4288 {
4289   neon_reinterpret (operands[0], operands[1]);
4290   DONE;
4291 })
4292
4293 (define_expand "neon_vreinterpretv2di<mode>"
4294   [(match_operand:V2DI 0 "s_register_operand" "")
4295    (match_operand:VQX 1 "s_register_operand" "")]
4296   "TARGET_NEON"
4297 {
4298   neon_reinterpret (operands[0], operands[1]);
4299   DONE;
4300 })
4301
4302 (define_expand "vec_load_lanes<mode><mode>"
4303   [(set (match_operand:VDQX 0 "s_register_operand")
4304         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4305                      UNSPEC_VLD1))]
4306   "TARGET_NEON")
4307
4308 (define_insn "neon_vld1<mode>"
4309   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4310         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4311                     UNSPEC_VLD1))]
4312   "TARGET_NEON"
4313   "vld1.<V_sz_elem>\t%h0, %A1"
4314   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4315 )
4316
4317 (define_insn "neon_vld1_lane<mode>"
4318   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4319         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4320                      (match_operand:VDX 2 "s_register_operand" "0")
4321                      (match_operand:SI 3 "immediate_operand" "i")]
4322                     UNSPEC_VLD1_LANE))]
4323   "TARGET_NEON"
4324 {
4325   HOST_WIDE_INT lane = INTVAL (operands[3]);
4326   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4327   if (lane < 0 || lane >= max)
4328     error ("lane out of range");
4329   if (max == 1)
4330     return "vld1.<V_sz_elem>\t%P0, %A1";
4331   else
4332     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4333 }
4334   [(set (attr "neon_type")
4335       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4336                     (const_string "neon_vld1_1_2_regs")
4337                     (const_string "neon_vld1_vld2_lane")))]
4338 )
4339
4340 (define_insn "neon_vld1_lane<mode>"
4341   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4342         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4343                      (match_operand:VQX 2 "s_register_operand" "0")
4344                      (match_operand:SI 3 "immediate_operand" "i")]
4345                     UNSPEC_VLD1_LANE))]
4346   "TARGET_NEON"
4347 {
4348   HOST_WIDE_INT lane = INTVAL (operands[3]);
4349   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4350   int regno = REGNO (operands[0]);
4351   if (lane < 0 || lane >= max)
4352     error ("lane out of range");
4353   else if (lane >= max / 2)
4354     {
4355       lane -= max / 2;
4356       regno += 2;
4357       operands[3] = GEN_INT (lane);
4358     }
4359   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4360   if (max == 2)
4361     return "vld1.<V_sz_elem>\t%P0, %A1";
4362   else
4363     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4364 }
4365   [(set (attr "neon_type")
4366       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4367                     (const_string "neon_vld1_1_2_regs")
4368                     (const_string "neon_vld1_vld2_lane")))]
4369 )
4370
4371 (define_insn "neon_vld1_dup<mode>"
4372   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4373         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4374   "TARGET_NEON"
4375 {
4376   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4377     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4378   else
4379     return "vld1.<V_sz_elem>\t%h0, %A1";
4380 }
4381   [(set (attr "neon_type")
4382       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4383                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4384                     (const_string "neon_vld1_1_2_regs")))]
4385 )
4386
4387 (define_insn "neon_vld1_dup<mode>"
4388   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4389         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4390   "TARGET_NEON"
4391 {
4392   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4393 }
4394   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4395 )
4396
4397 (define_insn_and_split "neon_vld1_dupv2di"
4398    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4399     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4400    "TARGET_NEON"
4401    "#"
4402    "&& reload_completed"
4403    [(const_int 0)]
4404    {
4405     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4406     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4407     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4408     DONE;
4409     }
4410   [(set_attr "length" "8")
4411    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4412 )
4413
4414 (define_expand "vec_store_lanes<mode><mode>"
4415   [(set (match_operand:VDQX 0 "neon_struct_operand")
4416         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4417                      UNSPEC_VST1))]
4418   "TARGET_NEON")
4419
4420 (define_insn "neon_vst1<mode>"
4421   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4422         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4423                      UNSPEC_VST1))]
4424   "TARGET_NEON"
4425   "vst1.<V_sz_elem>\t%h1, %A0"
4426   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4427
4428 (define_insn "neon_vst1_lane<mode>"
4429   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4430         (vec_select:<V_elem>
4431           (match_operand:VDX 1 "s_register_operand" "w")
4432           (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4433   "TARGET_NEON"
4434 {
4435   HOST_WIDE_INT lane = INTVAL (operands[2]);
4436   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4437   if (lane < 0 || lane >= max)
4438     error ("lane out of range");
4439   if (max == 1)
4440     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4441   else
4442     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4443 }
4444   [(set (attr "neon_type")
4445       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4446                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4447                     (const_string "neon_vst1_vst2_lane")))])
4448
4449 (define_insn "neon_vst1_lane<mode>"
4450   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4451         (vec_select:<V_elem>
4452            (match_operand:VQX 1 "s_register_operand" "w")
4453            (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4454   "TARGET_NEON"
4455 {
4456   HOST_WIDE_INT lane = INTVAL (operands[2]);
4457   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4458   int regno = REGNO (operands[1]);
4459   if (lane < 0 || lane >= max)
4460     error ("lane out of range");
4461   else if (lane >= max / 2)
4462     {
4463       lane -= max / 2;
4464       regno += 2;
4465       operands[2] = GEN_INT (lane);
4466     }
4467   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4468   if (max == 2)
4469     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4470   else
4471     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4472 }
4473   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4474 )
4475
4476 (define_expand "vec_load_lanesti<mode>"
4477   [(set (match_operand:TI 0 "s_register_operand")
4478         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4479                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4480                    UNSPEC_VLD2))]
4481   "TARGET_NEON")
4482
4483 (define_insn "neon_vld2<mode>"
4484   [(set (match_operand:TI 0 "s_register_operand" "=w")
4485         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4486                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4487                    UNSPEC_VLD2))]
4488   "TARGET_NEON"
4489 {
4490   if (<V_sz_elem> == 64)
4491     return "vld1.64\t%h0, %A1";
4492   else
4493     return "vld2.<V_sz_elem>\t%h0, %A1";
4494 }
4495   [(set (attr "neon_type")
4496       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4497                     (const_string "neon_vld1_1_2_regs")
4498                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4499 )
4500
4501 (define_expand "vec_load_lanesoi<mode>"
4502   [(set (match_operand:OI 0 "s_register_operand")
4503         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4504                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4505                    UNSPEC_VLD2))]
4506   "TARGET_NEON")
4507
4508 (define_insn "neon_vld2<mode>"
4509   [(set (match_operand:OI 0 "s_register_operand" "=w")
4510         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4511                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4512                    UNSPEC_VLD2))]
4513   "TARGET_NEON"
4514   "vld2.<V_sz_elem>\t%h0, %A1"
4515   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4516
4517 (define_insn "neon_vld2_lane<mode>"
4518   [(set (match_operand:TI 0 "s_register_operand" "=w")
4519         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4520                     (match_operand:TI 2 "s_register_operand" "0")
4521                     (match_operand:SI 3 "immediate_operand" "i")
4522                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4523                    UNSPEC_VLD2_LANE))]
4524   "TARGET_NEON"
4525 {
4526   HOST_WIDE_INT lane = INTVAL (operands[3]);
4527   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4528   int regno = REGNO (operands[0]);
4529   rtx ops[4];
4530   if (lane < 0 || lane >= max)
4531     error ("lane out of range");
4532   ops[0] = gen_rtx_REG (DImode, regno);
4533   ops[1] = gen_rtx_REG (DImode, regno + 2);
4534   ops[2] = operands[1];
4535   ops[3] = operands[3];
4536   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4537   return "";
4538 }
4539   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4540 )
4541
4542 (define_insn "neon_vld2_lane<mode>"
4543   [(set (match_operand:OI 0 "s_register_operand" "=w")
4544         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4545                     (match_operand:OI 2 "s_register_operand" "0")
4546                     (match_operand:SI 3 "immediate_operand" "i")
4547                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4548                    UNSPEC_VLD2_LANE))]
4549   "TARGET_NEON"
4550 {
4551   HOST_WIDE_INT lane = INTVAL (operands[3]);
4552   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4553   int regno = REGNO (operands[0]);
4554   rtx ops[4];
4555   if (lane < 0 || lane >= max)
4556     error ("lane out of range");
4557   else if (lane >= max / 2)
4558     {
4559       lane -= max / 2;
4560       regno += 2;
4561     }
4562   ops[0] = gen_rtx_REG (DImode, regno);
4563   ops[1] = gen_rtx_REG (DImode, regno + 4);
4564   ops[2] = operands[1];
4565   ops[3] = GEN_INT (lane);
4566   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4567   return "";
4568 }
4569   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4570 )
4571
4572 (define_insn "neon_vld2_dup<mode>"
4573   [(set (match_operand:TI 0 "s_register_operand" "=w")
4574         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4575                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4576                    UNSPEC_VLD2_DUP))]
4577   "TARGET_NEON"
4578 {
4579   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4580     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4581   else
4582     return "vld1.<V_sz_elem>\t%h0, %A1";
4583 }
4584   [(set (attr "neon_type")
4585       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4586                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4587                     (const_string "neon_vld1_1_2_regs")))]
4588 )
4589
4590 (define_expand "vec_store_lanesti<mode>"
4591   [(set (match_operand:TI 0 "neon_struct_operand")
4592         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4593                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4594                    UNSPEC_VST2))]
4595   "TARGET_NEON")
4596
4597 (define_insn "neon_vst2<mode>"
4598   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4599         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4600                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4601                    UNSPEC_VST2))]
4602   "TARGET_NEON"
4603 {
4604   if (<V_sz_elem> == 64)
4605     return "vst1.64\t%h1, %A0";
4606   else
4607     return "vst2.<V_sz_elem>\t%h1, %A0";
4608 }
4609   [(set (attr "neon_type")
4610       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4611                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4612                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4613 )
4614
4615 (define_expand "vec_store_lanesoi<mode>"
4616   [(set (match_operand:OI 0 "neon_struct_operand")
4617         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4618                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4619                    UNSPEC_VST2))]
4620   "TARGET_NEON")
4621
4622 (define_insn "neon_vst2<mode>"
4623   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4624         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4625                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4626                    UNSPEC_VST2))]
4627   "TARGET_NEON"
4628   "vst2.<V_sz_elem>\t%h1, %A0"
4629   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4630 )
4631
4632 (define_insn "neon_vst2_lane<mode>"
4633   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4634         (unspec:<V_two_elem>
4635           [(match_operand:TI 1 "s_register_operand" "w")
4636            (match_operand:SI 2 "immediate_operand" "i")
4637            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4638           UNSPEC_VST2_LANE))]
4639   "TARGET_NEON"
4640 {
4641   HOST_WIDE_INT lane = INTVAL (operands[2]);
4642   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4643   int regno = REGNO (operands[1]);
4644   rtx ops[4];
4645   if (lane < 0 || lane >= max)
4646     error ("lane out of range");
4647   ops[0] = operands[0];
4648   ops[1] = gen_rtx_REG (DImode, regno);
4649   ops[2] = gen_rtx_REG (DImode, regno + 2);
4650   ops[3] = operands[2];
4651   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4652   return "";
4653 }
4654   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4655 )
4656
4657 (define_insn "neon_vst2_lane<mode>"
4658   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4659         (unspec:<V_two_elem>
4660            [(match_operand:OI 1 "s_register_operand" "w")
4661             (match_operand:SI 2 "immediate_operand" "i")
4662             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4663            UNSPEC_VST2_LANE))]
4664   "TARGET_NEON"
4665 {
4666   HOST_WIDE_INT lane = INTVAL (operands[2]);
4667   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4668   int regno = REGNO (operands[1]);
4669   rtx ops[4];
4670   if (lane < 0 || lane >= max)
4671     error ("lane out of range");
4672   else if (lane >= max / 2)
4673     {
4674       lane -= max / 2;
4675       regno += 2;
4676     }
4677   ops[0] = operands[0];
4678   ops[1] = gen_rtx_REG (DImode, regno);
4679   ops[2] = gen_rtx_REG (DImode, regno + 4);
4680   ops[3] = GEN_INT (lane);
4681   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4682   return "";
4683 }
4684   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4685 )
4686
4687 (define_expand "vec_load_lanesei<mode>"
4688   [(set (match_operand:EI 0 "s_register_operand")
4689         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4690                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4691                    UNSPEC_VLD3))]
4692   "TARGET_NEON")
4693
4694 (define_insn "neon_vld3<mode>"
4695   [(set (match_operand:EI 0 "s_register_operand" "=w")
4696         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4697                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4698                    UNSPEC_VLD3))]
4699   "TARGET_NEON"
4700 {
4701   if (<V_sz_elem> == 64)
4702     return "vld1.64\t%h0, %A1";
4703   else
4704     return "vld3.<V_sz_elem>\t%h0, %A1";
4705 }
4706   [(set (attr "neon_type")
4707       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4708                     (const_string "neon_vld1_1_2_regs")
4709                     (const_string "neon_vld3_vld4")))]
4710 )
4711
4712 (define_expand "vec_load_lanesci<mode>"
4713   [(match_operand:CI 0 "s_register_operand")
4714    (match_operand:CI 1 "neon_struct_operand")
4715    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4716   "TARGET_NEON"
4717 {
4718   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4719   DONE;
4720 })
4721
4722 (define_expand "neon_vld3<mode>"
4723   [(match_operand:CI 0 "s_register_operand")
4724    (match_operand:CI 1 "neon_struct_operand")
4725    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4726   "TARGET_NEON"
4727 {
4728   rtx mem;
4729
4730   mem = adjust_address (operands[1], EImode, 0);
4731   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4732   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4733   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4734   DONE;
4735 })
4736
4737 (define_insn "neon_vld3qa<mode>"
4738   [(set (match_operand:CI 0 "s_register_operand" "=w")
4739         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4740                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4741                    UNSPEC_VLD3A))]
4742   "TARGET_NEON"
4743 {
4744   int regno = REGNO (operands[0]);
4745   rtx ops[4];
4746   ops[0] = gen_rtx_REG (DImode, regno);
4747   ops[1] = gen_rtx_REG (DImode, regno + 4);
4748   ops[2] = gen_rtx_REG (DImode, regno + 8);
4749   ops[3] = operands[1];
4750   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4751   return "";
4752 }
4753   [(set_attr "neon_type" "neon_vld3_vld4")]
4754 )
4755
4756 (define_insn "neon_vld3qb<mode>"
4757   [(set (match_operand:CI 0 "s_register_operand" "=w")
4758         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4759                     (match_operand:CI 2 "s_register_operand" "0")
4760                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4761                    UNSPEC_VLD3B))]
4762   "TARGET_NEON"
4763 {
4764   int regno = REGNO (operands[0]);
4765   rtx ops[4];
4766   ops[0] = gen_rtx_REG (DImode, regno + 2);
4767   ops[1] = gen_rtx_REG (DImode, regno + 6);
4768   ops[2] = gen_rtx_REG (DImode, regno + 10);
4769   ops[3] = operands[1];
4770   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4771   return "";
4772 }
4773   [(set_attr "neon_type" "neon_vld3_vld4")]
4774 )
4775
4776 (define_insn "neon_vld3_lane<mode>"
4777   [(set (match_operand:EI 0 "s_register_operand" "=w")
4778         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4779                     (match_operand:EI 2 "s_register_operand" "0")
4780                     (match_operand:SI 3 "immediate_operand" "i")
4781                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4782                    UNSPEC_VLD3_LANE))]
4783   "TARGET_NEON"
4784 {
4785   HOST_WIDE_INT lane = INTVAL (operands[3]);
4786   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4787   int regno = REGNO (operands[0]);
4788   rtx ops[5];
4789   if (lane < 0 || lane >= max)
4790     error ("lane out of range");
4791   ops[0] = gen_rtx_REG (DImode, regno);
4792   ops[1] = gen_rtx_REG (DImode, regno + 2);
4793   ops[2] = gen_rtx_REG (DImode, regno + 4);
4794   ops[3] = operands[1];
4795   ops[4] = operands[3];
4796   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4797                    ops);
4798   return "";
4799 }
4800   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4801 )
4802
4803 (define_insn "neon_vld3_lane<mode>"
4804   [(set (match_operand:CI 0 "s_register_operand" "=w")
4805         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4806                     (match_operand:CI 2 "s_register_operand" "0")
4807                     (match_operand:SI 3 "immediate_operand" "i")
4808                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4809                    UNSPEC_VLD3_LANE))]
4810   "TARGET_NEON"
4811 {
4812   HOST_WIDE_INT lane = INTVAL (operands[3]);
4813   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4814   int regno = REGNO (operands[0]);
4815   rtx ops[5];
4816   if (lane < 0 || lane >= max)
4817     error ("lane out of range");
4818   else if (lane >= max / 2)
4819     {
4820       lane -= max / 2;
4821       regno += 2;
4822     }
4823   ops[0] = gen_rtx_REG (DImode, regno);
4824   ops[1] = gen_rtx_REG (DImode, regno + 4);
4825   ops[2] = gen_rtx_REG (DImode, regno + 8);
4826   ops[3] = operands[1];
4827   ops[4] = GEN_INT (lane);
4828   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4829                    ops);
4830   return "";
4831 }
4832   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4833 )
4834
4835 (define_insn "neon_vld3_dup<mode>"
4836   [(set (match_operand:EI 0 "s_register_operand" "=w")
4837         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4838                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4839                    UNSPEC_VLD3_DUP))]
4840   "TARGET_NEON"
4841 {
4842   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4843     {
4844       int regno = REGNO (operands[0]);
4845       rtx ops[4];
4846       ops[0] = gen_rtx_REG (DImode, regno);
4847       ops[1] = gen_rtx_REG (DImode, regno + 2);
4848       ops[2] = gen_rtx_REG (DImode, regno + 4);
4849       ops[3] = operands[1];
4850       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
4851       return "";
4852     }
4853   else
4854     return "vld1.<V_sz_elem>\t%h0, %A1";
4855 }
4856   [(set (attr "neon_type")
4857       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4858                     (const_string "neon_vld3_vld4_all_lanes")
4859                     (const_string "neon_vld1_1_2_regs")))])
4860
4861 (define_expand "vec_store_lanesei<mode>"
4862   [(set (match_operand:EI 0 "neon_struct_operand")
4863         (unspec:EI [(match_operand:EI 1 "s_register_operand")
4864                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4865                    UNSPEC_VST3))]
4866   "TARGET_NEON")
4867
4868 (define_insn "neon_vst3<mode>"
4869   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4870         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4871                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4872                    UNSPEC_VST3))]
4873   "TARGET_NEON"
4874 {
4875   if (<V_sz_elem> == 64)
4876     return "vst1.64\t%h1, %A0";
4877   else
4878     return "vst3.<V_sz_elem>\t%h1, %A0";
4879 }
4880   [(set (attr "neon_type")
4881       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4882                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4883                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
4884
4885 (define_expand "vec_store_lanesci<mode>"
4886   [(match_operand:CI 0 "neon_struct_operand")
4887    (match_operand:CI 1 "s_register_operand")
4888    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4889   "TARGET_NEON"
4890 {
4891   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4892   DONE;
4893 })
4894
4895 (define_expand "neon_vst3<mode>"
4896   [(match_operand:CI 0 "neon_struct_operand")
4897    (match_operand:CI 1 "s_register_operand")
4898    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4899   "TARGET_NEON"
4900 {
4901   rtx mem;
4902
4903   mem = adjust_address (operands[0], EImode, 0);
4904   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4905   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4906   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
4907   DONE;
4908 })
4909
4910 (define_insn "neon_vst3qa<mode>"
4911   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4912         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4913                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4914                    UNSPEC_VST3A))]
4915   "TARGET_NEON"
4916 {
4917   int regno = REGNO (operands[1]);
4918   rtx ops[4];
4919   ops[0] = operands[0];
4920   ops[1] = gen_rtx_REG (DImode, regno);
4921   ops[2] = gen_rtx_REG (DImode, regno + 4);
4922   ops[3] = gen_rtx_REG (DImode, regno + 8);
4923   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4924   return "";
4925 }
4926   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4927 )
4928
4929 (define_insn "neon_vst3qb<mode>"
4930   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4931         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4932                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4933                    UNSPEC_VST3B))]
4934   "TARGET_NEON"
4935 {
4936   int regno = REGNO (operands[1]);
4937   rtx ops[4];
4938   ops[0] = operands[0];
4939   ops[1] = gen_rtx_REG (DImode, regno + 2);
4940   ops[2] = gen_rtx_REG (DImode, regno + 6);
4941   ops[3] = gen_rtx_REG (DImode, regno + 10);
4942   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4943   return "";
4944 }
4945   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4946 )
4947
4948 (define_insn "neon_vst3_lane<mode>"
4949   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4950         (unspec:<V_three_elem>
4951            [(match_operand:EI 1 "s_register_operand" "w")
4952             (match_operand:SI 2 "immediate_operand" "i")
4953             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4954            UNSPEC_VST3_LANE))]
4955   "TARGET_NEON"
4956 {
4957   HOST_WIDE_INT lane = INTVAL (operands[2]);
4958   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4959   int regno = REGNO (operands[1]);
4960   rtx ops[5];
4961   if (lane < 0 || lane >= max)
4962     error ("lane out of range");
4963   ops[0] = operands[0];
4964   ops[1] = gen_rtx_REG (DImode, regno);
4965   ops[2] = gen_rtx_REG (DImode, regno + 2);
4966   ops[3] = gen_rtx_REG (DImode, regno + 4);
4967   ops[4] = operands[2];
4968   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
4969                    ops);
4970   return "";
4971 }
4972   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4973 )
4974
4975 (define_insn "neon_vst3_lane<mode>"
4976   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4977         (unspec:<V_three_elem>
4978            [(match_operand:CI 1 "s_register_operand" "w")
4979             (match_operand:SI 2 "immediate_operand" "i")
4980             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4981            UNSPEC_VST3_LANE))]
4982   "TARGET_NEON"
4983 {
4984   HOST_WIDE_INT lane = INTVAL (operands[2]);
4985   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4986   int regno = REGNO (operands[1]);
4987   rtx ops[5];
4988   if (lane < 0 || lane >= max)
4989     error ("lane out of range");
4990   else if (lane >= max / 2)
4991     {
4992       lane -= max / 2;
4993       regno += 2;
4994     }
4995   ops[0] = operands[0];
4996   ops[1] = gen_rtx_REG (DImode, regno);
4997   ops[2] = gen_rtx_REG (DImode, regno + 4);
4998   ops[3] = gen_rtx_REG (DImode, regno + 8);
4999   ops[4] = GEN_INT (lane);
5000   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5001                    ops);
5002   return "";
5003 }
5004 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5005
5006 (define_expand "vec_load_lanesoi<mode>"
5007   [(set (match_operand:OI 0 "s_register_operand")
5008         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5009                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5010                    UNSPEC_VLD4))]
5011   "TARGET_NEON")
5012
5013 (define_insn "neon_vld4<mode>"
5014   [(set (match_operand:OI 0 "s_register_operand" "=w")
5015         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5016                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5017                    UNSPEC_VLD4))]
5018   "TARGET_NEON"
5019 {
5020   if (<V_sz_elem> == 64)
5021     return "vld1.64\t%h0, %A1";
5022   else
5023     return "vld4.<V_sz_elem>\t%h0, %A1";
5024 }
5025   [(set (attr "neon_type")
5026       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5027                     (const_string "neon_vld1_1_2_regs")
5028                     (const_string "neon_vld3_vld4")))]
5029 )
5030
5031 (define_expand "vec_load_lanesxi<mode>"
5032   [(match_operand:XI 0 "s_register_operand")
5033    (match_operand:XI 1 "neon_struct_operand")
5034    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5035   "TARGET_NEON"
5036 {
5037   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5038   DONE;
5039 })
5040
5041 (define_expand "neon_vld4<mode>"
5042   [(match_operand:XI 0 "s_register_operand")
5043    (match_operand:XI 1 "neon_struct_operand")
5044    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5045   "TARGET_NEON"
5046 {
5047   rtx mem;
5048
5049   mem = adjust_address (operands[1], OImode, 0);
5050   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5051   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5052   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5053   DONE;
5054 })
5055
5056 (define_insn "neon_vld4qa<mode>"
5057   [(set (match_operand:XI 0 "s_register_operand" "=w")
5058         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5059                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5060                    UNSPEC_VLD4A))]
5061   "TARGET_NEON"
5062 {
5063   int regno = REGNO (operands[0]);
5064   rtx ops[5];
5065   ops[0] = gen_rtx_REG (DImode, regno);
5066   ops[1] = gen_rtx_REG (DImode, regno + 4);
5067   ops[2] = gen_rtx_REG (DImode, regno + 8);
5068   ops[3] = gen_rtx_REG (DImode, regno + 12);
5069   ops[4] = operands[1];
5070   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5071   return "";
5072 }
5073   [(set_attr "neon_type" "neon_vld3_vld4")]
5074 )
5075
5076 (define_insn "neon_vld4qb<mode>"
5077   [(set (match_operand:XI 0 "s_register_operand" "=w")
5078         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5079                     (match_operand:XI 2 "s_register_operand" "0")
5080                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5081                    UNSPEC_VLD4B))]
5082   "TARGET_NEON"
5083 {
5084   int regno = REGNO (operands[0]);
5085   rtx ops[5];
5086   ops[0] = gen_rtx_REG (DImode, regno + 2);
5087   ops[1] = gen_rtx_REG (DImode, regno + 6);
5088   ops[2] = gen_rtx_REG (DImode, regno + 10);
5089   ops[3] = gen_rtx_REG (DImode, regno + 14);
5090   ops[4] = operands[1];
5091   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5092   return "";
5093 }
5094   [(set_attr "neon_type" "neon_vld3_vld4")]
5095 )
5096
5097 (define_insn "neon_vld4_lane<mode>"
5098   [(set (match_operand:OI 0 "s_register_operand" "=w")
5099         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5100                     (match_operand:OI 2 "s_register_operand" "0")
5101                     (match_operand:SI 3 "immediate_operand" "i")
5102                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5103                    UNSPEC_VLD4_LANE))]
5104   "TARGET_NEON"
5105 {
5106   HOST_WIDE_INT lane = INTVAL (operands[3]);
5107   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5108   int regno = REGNO (operands[0]);
5109   rtx ops[6];
5110   if (lane < 0 || lane >= max)
5111     error ("lane out of range");
5112   ops[0] = gen_rtx_REG (DImode, regno);
5113   ops[1] = gen_rtx_REG (DImode, regno + 2);
5114   ops[2] = gen_rtx_REG (DImode, regno + 4);
5115   ops[3] = gen_rtx_REG (DImode, regno + 6);
5116   ops[4] = operands[1];
5117   ops[5] = operands[3];
5118   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5119                    ops);
5120   return "";
5121 }
5122   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5123 )
5124
5125 (define_insn "neon_vld4_lane<mode>"
5126   [(set (match_operand:XI 0 "s_register_operand" "=w")
5127         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5128                     (match_operand:XI 2 "s_register_operand" "0")
5129                     (match_operand:SI 3 "immediate_operand" "i")
5130                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5131                    UNSPEC_VLD4_LANE))]
5132   "TARGET_NEON"
5133 {
5134   HOST_WIDE_INT lane = INTVAL (operands[3]);
5135   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5136   int regno = REGNO (operands[0]);
5137   rtx ops[6];
5138   if (lane < 0 || lane >= max)
5139     error ("lane out of range");
5140   else if (lane >= max / 2)
5141     {
5142       lane -= max / 2;
5143       regno += 2;
5144     }
5145   ops[0] = gen_rtx_REG (DImode, regno);
5146   ops[1] = gen_rtx_REG (DImode, regno + 4);
5147   ops[2] = gen_rtx_REG (DImode, regno + 8);
5148   ops[3] = gen_rtx_REG (DImode, regno + 12);
5149   ops[4] = operands[1];
5150   ops[5] = GEN_INT (lane);
5151   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5152                    ops);
5153   return "";
5154 }
5155   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5156 )
5157
5158 (define_insn "neon_vld4_dup<mode>"
5159   [(set (match_operand:OI 0 "s_register_operand" "=w")
5160         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5161                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5162                    UNSPEC_VLD4_DUP))]
5163   "TARGET_NEON"
5164 {
5165   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5166     {
5167       int regno = REGNO (operands[0]);
5168       rtx ops[5];
5169       ops[0] = gen_rtx_REG (DImode, regno);
5170       ops[1] = gen_rtx_REG (DImode, regno + 2);
5171       ops[2] = gen_rtx_REG (DImode, regno + 4);
5172       ops[3] = gen_rtx_REG (DImode, regno + 6);
5173       ops[4] = operands[1];
5174       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5175                        ops);
5176       return "";
5177     }
5178   else
5179     return "vld1.<V_sz_elem>\t%h0, %A1";
5180 }
5181   [(set (attr "neon_type")
5182       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5183                     (const_string "neon_vld3_vld4_all_lanes")
5184                     (const_string "neon_vld1_1_2_regs")))]
5185 )
5186
5187 (define_expand "vec_store_lanesoi<mode>"
5188   [(set (match_operand:OI 0 "neon_struct_operand")
5189         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5190                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5191                    UNSPEC_VST4))]
5192   "TARGET_NEON")
5193
5194 (define_insn "neon_vst4<mode>"
5195   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5196         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5197                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5198                    UNSPEC_VST4))]
5199   "TARGET_NEON"
5200 {
5201   if (<V_sz_elem> == 64)
5202     return "vst1.64\t%h1, %A0";
5203   else
5204     return "vst4.<V_sz_elem>\t%h1, %A0";
5205 }
5206   [(set (attr "neon_type")
5207       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5208                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5209                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5210 )
5211
5212 (define_expand "vec_store_lanesxi<mode>"
5213   [(match_operand:XI 0 "neon_struct_operand")
5214    (match_operand:XI 1 "s_register_operand")
5215    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5216   "TARGET_NEON"
5217 {
5218   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5219   DONE;
5220 })
5221
5222 (define_expand "neon_vst4<mode>"
5223   [(match_operand:XI 0 "neon_struct_operand")
5224    (match_operand:XI 1 "s_register_operand")
5225    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5226   "TARGET_NEON"
5227 {
5228   rtx mem;
5229
5230   mem = adjust_address (operands[0], OImode, 0);
5231   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5232   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5233   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5234   DONE;
5235 })
5236
5237 (define_insn "neon_vst4qa<mode>"
5238   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5239         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5240                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5241                    UNSPEC_VST4A))]
5242   "TARGET_NEON"
5243 {
5244   int regno = REGNO (operands[1]);
5245   rtx ops[5];
5246   ops[0] = operands[0];
5247   ops[1] = gen_rtx_REG (DImode, regno);
5248   ops[2] = gen_rtx_REG (DImode, regno + 4);
5249   ops[3] = gen_rtx_REG (DImode, regno + 8);
5250   ops[4] = gen_rtx_REG (DImode, regno + 12);
5251   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5252   return "";
5253 }
5254   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5255 )
5256
5257 (define_insn "neon_vst4qb<mode>"
5258   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5259         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5260                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5261                    UNSPEC_VST4B))]
5262   "TARGET_NEON"
5263 {
5264   int regno = REGNO (operands[1]);
5265   rtx ops[5];
5266   ops[0] = operands[0];
5267   ops[1] = gen_rtx_REG (DImode, regno + 2);
5268   ops[2] = gen_rtx_REG (DImode, regno + 6);
5269   ops[3] = gen_rtx_REG (DImode, regno + 10);
5270   ops[4] = gen_rtx_REG (DImode, regno + 14);
5271   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5272   return "";
5273 }
5274   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5275 )
5276
5277 (define_insn "neon_vst4_lane<mode>"
5278   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5279         (unspec:<V_four_elem>
5280            [(match_operand:OI 1 "s_register_operand" "w")
5281             (match_operand:SI 2 "immediate_operand" "i")
5282             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5283            UNSPEC_VST4_LANE))]
5284   "TARGET_NEON"
5285 {
5286   HOST_WIDE_INT lane = INTVAL (operands[2]);
5287   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5288   int regno = REGNO (operands[1]);
5289   rtx ops[6];
5290   if (lane < 0 || lane >= max)
5291     error ("lane out of range");
5292   ops[0] = operands[0];
5293   ops[1] = gen_rtx_REG (DImode, regno);
5294   ops[2] = gen_rtx_REG (DImode, regno + 2);
5295   ops[3] = gen_rtx_REG (DImode, regno + 4);
5296   ops[4] = gen_rtx_REG (DImode, regno + 6);
5297   ops[5] = operands[2];
5298   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5299                    ops);
5300   return "";
5301 }
5302   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5303 )
5304
5305 (define_insn "neon_vst4_lane<mode>"
5306   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5307         (unspec:<V_four_elem>
5308            [(match_operand:XI 1 "s_register_operand" "w")
5309             (match_operand:SI 2 "immediate_operand" "i")
5310             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5311            UNSPEC_VST4_LANE))]
5312   "TARGET_NEON"
5313 {
5314   HOST_WIDE_INT lane = INTVAL (operands[2]);
5315   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5316   int regno = REGNO (operands[1]);
5317   rtx ops[6];
5318   if (lane < 0 || lane >= max)
5319     error ("lane out of range");
5320   else if (lane >= max / 2)
5321     {
5322       lane -= max / 2;
5323       regno += 2;
5324     }
5325   ops[0] = operands[0];
5326   ops[1] = gen_rtx_REG (DImode, regno);
5327   ops[2] = gen_rtx_REG (DImode, regno + 4);
5328   ops[3] = gen_rtx_REG (DImode, regno + 8);
5329   ops[4] = gen_rtx_REG (DImode, regno + 12);
5330   ops[5] = GEN_INT (lane);
5331   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5332                    ops);
5333   return "";
5334 }
5335   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5336 )
5337
5338 (define_expand "neon_vand<mode>"
5339   [(match_operand:VDQX 0 "s_register_operand" "")
5340    (match_operand:VDQX 1 "s_register_operand" "")
5341    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5342    (match_operand:SI 3 "immediate_operand" "")]
5343   "TARGET_NEON"
5344 {
5345   emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5346   DONE;
5347 })
5348
5349 (define_expand "neon_vorr<mode>"
5350   [(match_operand:VDQX 0 "s_register_operand" "")
5351    (match_operand:VDQX 1 "s_register_operand" "")
5352    (match_operand:VDQX 2 "neon_logic_op2" "")
5353    (match_operand:SI 3 "immediate_operand" "")]
5354   "TARGET_NEON"
5355 {
5356   emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5357   DONE;
5358 })
5359
5360 (define_expand "neon_veor<mode>"
5361   [(match_operand:VDQX 0 "s_register_operand" "")
5362    (match_operand:VDQX 1 "s_register_operand" "")
5363    (match_operand:VDQX 2 "s_register_operand" "")
5364    (match_operand:SI 3 "immediate_operand" "")]
5365   "TARGET_NEON"
5366 {
5367   emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5368   DONE;
5369 })
5370
5371 (define_expand "neon_vbic<mode>"
5372   [(match_operand:VDQX 0 "s_register_operand" "")
5373    (match_operand:VDQX 1 "s_register_operand" "")
5374    (match_operand:VDQX 2 "neon_logic_op2" "")
5375    (match_operand:SI 3 "immediate_operand" "")]
5376   "TARGET_NEON"
5377 {
5378   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5379   DONE;
5380 })
5381
5382 (define_expand "neon_vorn<mode>"
5383   [(match_operand:VDQX 0 "s_register_operand" "")
5384    (match_operand:VDQX 1 "s_register_operand" "")
5385    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5386    (match_operand:SI 3 "immediate_operand" "")]
5387   "TARGET_NEON"
5388 {
5389   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5390   DONE;
5391 })
5392
5393 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5394   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5395         (SE:<V_unpack> (vec_select:<V_HALF>
5396                           (match_operand:VU 1 "register_operand" "w")
5397                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5398   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5399   "vmovl.<US><V_sz_elem> %q0, %e1"
5400   [(set_attr "neon_type" "neon_shift_1")]
5401 )
5402
5403 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5404   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5405         (SE:<V_unpack> (vec_select:<V_HALF>
5406                           (match_operand:VU 1 "register_operand" "w")
5407                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5408   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5409   "vmovl.<US><V_sz_elem> %q0, %f1"
5410   [(set_attr "neon_type" "neon_shift_1")]
5411 )
5412
5413 (define_expand "vec_unpack<US>_hi_<mode>"
5414   [(match_operand:<V_unpack> 0 "register_operand" "")
5415    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5416  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5417   {
5418    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5419    rtx t1;
5420    int i;
5421    for (i = 0; i < (<V_mode_nunits>/2); i++)
5422      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5423
5424    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5425    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5426                                                  operands[1],
5427                                                  t1));
5428    DONE;
5429   }
5430 )
5431
5432 (define_expand "vec_unpack<US>_lo_<mode>"
5433   [(match_operand:<V_unpack> 0 "register_operand" "")
5434    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5435  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5436   {
5437    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5438    rtx t1;
5439    int i;
5440    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5441      RTVEC_ELT (v, i) = GEN_INT (i);
5442    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5443    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5444                                                  operands[1],
5445                                                  t1));
5446    DONE;
5447   }
5448 )
5449
5450 (define_insn "neon_vec_<US>mult_lo_<mode>"
5451  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5452        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5453                            (match_operand:VU 1 "register_operand" "w")
5454                            (match_operand:VU 2 "vect_par_constant_low" "")))
5455                         (SE:<V_unpack> (vec_select:<V_HALF>
5456                            (match_operand:VU 3 "register_operand" "w")
5457                            (match_dup 2)))))]
5458   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5459   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5460   [(set_attr "neon_type" "neon_shift_1")]
5461 )
5462
5463 (define_expand "vec_widen_<US>mult_lo_<mode>"
5464   [(match_operand:<V_unpack> 0 "register_operand" "")
5465    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5466    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5467  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5468  {
5469    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5470    rtx t1;
5471    int i;
5472    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5473      RTVEC_ELT (v, i) = GEN_INT (i);
5474    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5475
5476    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5477                                                operands[1],
5478                                                t1,
5479                                                operands[2]));
5480    DONE;
5481  }
5482 )
5483
5484 (define_insn "neon_vec_<US>mult_hi_<mode>"
5485  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5486       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5487                             (match_operand:VU 1 "register_operand" "w")
5488                             (match_operand:VU 2 "vect_par_constant_high" "")))
5489                        (SE:<V_unpack> (vec_select:<V_HALF>
5490                             (match_operand:VU 3 "register_operand" "w")
5491                             (match_dup 2)))))]
5492   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5493   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5494   [(set_attr "neon_type" "neon_shift_1")]
5495 )
5496
5497 (define_expand "vec_widen_<US>mult_hi_<mode>"
5498   [(match_operand:<V_unpack> 0 "register_operand" "")
5499    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5500    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5501  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5502  {
5503    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5504    rtx t1;
5505    int i;
5506    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5507      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5508    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5509
5510    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5511                                                operands[1],
5512                                                t1,
5513                                                operands[2]));
5514    DONE;
5515
5516  }
5517 )
5518
5519 (define_insn "neon_vec_<US>shiftl_<mode>"
5520  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5521        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5522        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5523   "TARGET_NEON"
5524 {
5525   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5526 }
5527   [(set_attr "neon_type" "neon_shift_1")]
5528 )
5529
5530 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5531   [(match_operand:<V_unpack> 0 "register_operand" "")
5532    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5533    (match_operand:SI 2 "immediate_operand" "i")]
5534  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5535  {
5536   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5537                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5538                 operands[2]));
5539    DONE;
5540  }
5541 )
5542
5543 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5544   [(match_operand:<V_unpack> 0 "register_operand" "")
5545    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5546    (match_operand:SI 2 "immediate_operand" "i")]
5547  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5548  {
5549   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5550                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5551                                      GET_MODE_SIZE (<V_HALF>mode)),
5552                 operands[2]));
5553    DONE;
5554  }
5555 )
5556
5557 ;; Vectorize for non-neon-quad case
5558 (define_insn "neon_unpack<US>_<mode>"
5559  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5560        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5561  "TARGET_NEON"
5562  "vmovl.<US><V_sz_elem> %q0, %P1"
5563   [(set_attr "neon_type" "neon_shift_1")]
5564 )
5565
5566 (define_expand "vec_unpack<US>_lo_<mode>"
5567  [(match_operand:<V_double_width> 0 "register_operand" "")
5568   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5569  "TARGET_NEON"
5570 {
5571   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5572   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5573   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5574
5575   DONE;
5576 }
5577 )
5578
5579 (define_expand "vec_unpack<US>_hi_<mode>"
5580  [(match_operand:<V_double_width> 0 "register_operand" "")
5581   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5582  "TARGET_NEON"
5583 {
5584   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5585   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5586   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5587
5588   DONE;
5589 }
5590 )
5591
5592 (define_insn "neon_vec_<US>mult_<mode>"
5593  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5594        (mult:<V_widen> (SE:<V_widen>
5595                            (match_operand:VDI 1 "register_operand" "w"))
5596                        (SE:<V_widen>
5597                            (match_operand:VDI 2 "register_operand" "w"))))]
5598   "TARGET_NEON"
5599   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5600   [(set_attr "neon_type" "neon_shift_1")]
5601 )
5602
5603 (define_expand "vec_widen_<US>mult_hi_<mode>"
5604   [(match_operand:<V_double_width> 0 "register_operand" "")
5605    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5606    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5607  "TARGET_NEON"
5608  {
5609    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5610    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5611    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5612
5613    DONE;
5614
5615  }
5616 )
5617
5618 (define_expand "vec_widen_<US>mult_lo_<mode>"
5619   [(match_operand:<V_double_width> 0 "register_operand" "")
5620    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5621    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5622  "TARGET_NEON"
5623  {
5624    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5625    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5626    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5627
5628    DONE;
5629
5630  }
5631 )
5632
5633 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5634  [(match_operand:<V_double_width> 0 "register_operand" "")
5635    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5636    (match_operand:SI 2 "immediate_operand" "i")]
5637  "TARGET_NEON"
5638  {
5639    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5640    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5641    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5642
5643    DONE;
5644  }
5645 )
5646
5647 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5648   [(match_operand:<V_double_width> 0 "register_operand" "")
5649    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5650    (match_operand:SI 2 "immediate_operand" "i")]
5651  "TARGET_NEON"
5652  {
5653    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5654    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5655    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5656
5657    DONE;
5658  }
5659 )
5660
5661 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5662 ; because the ordering of vector elements in Q registers is different from what
5663 ; the semantics of the instructions require.
5664
5665 (define_insn "vec_pack_trunc_<mode>"
5666  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5667        (vec_concat:<V_narrow_pack>
5668                 (truncate:<V_narrow>
5669                         (match_operand:VN 1 "register_operand" "w"))
5670                 (truncate:<V_narrow>
5671                         (match_operand:VN 2 "register_operand" "w"))))]
5672  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5673  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5674  [(set_attr "neon_type" "neon_shift_1")
5675   (set_attr "length" "8")]
5676 )
5677
5678 ;; For the non-quad case.
5679 (define_insn "neon_vec_pack_trunc_<mode>"
5680  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5681        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5682  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5683  "vmovn.i<V_sz_elem>\t%P0, %q1"
5684  [(set_attr "neon_type" "neon_shift_1")]
5685 )
5686
5687 (define_expand "vec_pack_trunc_<mode>"
5688  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5689   (match_operand:VSHFT 1 "register_operand" "")
5690   (match_operand:VSHFT 2 "register_operand")]
5691  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5692 {
5693   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5694
5695   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5696   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5697   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5698   DONE;
5699 })
5700
5701 (define_insn "neon_vabd<mode>_2"
5702  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5703        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5704                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5705  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5706  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5707  [(set (attr "neon_type")
5708        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5709                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5710                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5711                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5712                      (const_string "neon_int_5")))]
5713 )
5714
5715 (define_insn "neon_vabd<mode>_3"
5716  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5717        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5718                              (match_operand:VDQ 2 "s_register_operand" "w")]
5719                  UNSPEC_VSUB)))]
5720  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5721  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5722  [(set (attr "neon_type")
5723        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5724                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5725                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5726                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5727                      (const_string "neon_int_5")))]
5728 )