gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
   3 ;;   Free Software Foundation, Inc.
   4 ;; Written by CodeSourcery.
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published by
  10 ;; the Free Software Foundation; either version 3, or (at your option)
  11 ;; any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but
  14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 ;; General Public License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22 ;; Enumerators for unspecs.
  23 (define_c_enum "unspec" [
  24   UNSPEC_ASHIFT_SIGNED
  25   UNSPEC_ASHIFT_UNSIGNED
  26   UNSPEC_VABD
  27   UNSPEC_VABDL
  28   UNSPEC_VADD
  29   UNSPEC_VADDHN
  30   UNSPEC_VADDL
  31   UNSPEC_VADDW
  32   UNSPEC_VBSL
  33   UNSPEC_VCAGE
  34   UNSPEC_VCAGT
  35   UNSPEC_VCEQ
  36   UNSPEC_VCGE
  37   UNSPEC_VCGEU
  38   UNSPEC_VCGT
  39   UNSPEC_VCGTU
  40   UNSPEC_VCLS
  41   UNSPEC_VCONCAT
  42   UNSPEC_VCVT
  43   UNSPEC_VCVT_N
  44   UNSPEC_VEXT
  45   UNSPEC_VHADD
  46   UNSPEC_VHSUB
  47   UNSPEC_VLD1
  48   UNSPEC_VLD1_LANE
  49   UNSPEC_VLD2
  50   UNSPEC_VLD2_DUP
  51   UNSPEC_VLD2_LANE
  52   UNSPEC_VLD3
  53   UNSPEC_VLD3A
  54   UNSPEC_VLD3B
  55   UNSPEC_VLD3_DUP
  56   UNSPEC_VLD3_LANE
  57   UNSPEC_VLD4
  58   UNSPEC_VLD4A
  59   UNSPEC_VLD4B
  60   UNSPEC_VLD4_DUP
  61   UNSPEC_VLD4_LANE
  62   UNSPEC_VMAX
  63   UNSPEC_VMIN
  64   UNSPEC_VMLA
  65   UNSPEC_VMLAL
  66   UNSPEC_VMLA_LANE
  67   UNSPEC_VMLAL_LANE
  68   UNSPEC_VMLS
  69   UNSPEC_VMLSL
  70   UNSPEC_VMLS_LANE
  71   UNSPEC_VMLSL_LANE
  72   UNSPEC_VMOVL
  73   UNSPEC_VMOVN
  74   UNSPEC_VMUL
  75   UNSPEC_VMULL
  76   UNSPEC_VMUL_LANE
  77   UNSPEC_VMULL_LANE
  78   UNSPEC_VPADAL
  79   UNSPEC_VPADD
  80   UNSPEC_VPADDL
  81   UNSPEC_VPMAX
  82   UNSPEC_VPMIN
  83   UNSPEC_VPSMAX
  84   UNSPEC_VPSMIN
  85   UNSPEC_VPUMAX
  86   UNSPEC_VPUMIN
  87   UNSPEC_VQABS
  88   UNSPEC_VQADD
  89   UNSPEC_VQDMLAL
  90   UNSPEC_VQDMLAL_LANE
  91   UNSPEC_VQDMLSL
  92   UNSPEC_VQDMLSL_LANE
  93   UNSPEC_VQDMULH
  94   UNSPEC_VQDMULH_LANE
  95   UNSPEC_VQDMULL
  96   UNSPEC_VQDMULL_LANE
  97   UNSPEC_VQMOVN
  98   UNSPEC_VQMOVUN
  99   UNSPEC_VQNEG
 100   UNSPEC_VQSHL
 101   UNSPEC_VQSHL_N
 102   UNSPEC_VQSHLU_N
 103   UNSPEC_VQSHRN_N
 104   UNSPEC_VQSHRUN_N
 105   UNSPEC_VQSUB
 106   UNSPEC_VRECPE
 107   UNSPEC_VRECPS
 108   UNSPEC_VREV16
 109   UNSPEC_VREV32
 110   UNSPEC_VREV64
 111   UNSPEC_VRSQRTE
 112   UNSPEC_VRSQRTS
 113   UNSPEC_VSHL
 114   UNSPEC_VSHLL_N
 115   UNSPEC_VSHL_N
 116   UNSPEC_VSHR_N
 117   UNSPEC_VSHRN_N
 118   UNSPEC_VSLI
 119   UNSPEC_VSRA_N
 120   UNSPEC_VSRI
 121   UNSPEC_VST1
 122   UNSPEC_VST1_LANE
 123   UNSPEC_VST2
 124   UNSPEC_VST2_LANE
 125   UNSPEC_VST3
 126   UNSPEC_VST3A
 127   UNSPEC_VST3B
 128   UNSPEC_VST3_LANE
 129   UNSPEC_VST4
 130   UNSPEC_VST4A
 131   UNSPEC_VST4B
 132   UNSPEC_VST4_LANE
 133   UNSPEC_VSTRUCTDUMMY
 134   UNSPEC_VSUB
 135   UNSPEC_VSUBHN
 136   UNSPEC_VSUBL
 137   UNSPEC_VSUBW
 138   UNSPEC_VTBL
 139   UNSPEC_VTBX
 140   UNSPEC_VTRN1
 141   UNSPEC_VTRN2
 142   UNSPEC_VTST
 143   UNSPEC_VUZP1
 144   UNSPEC_VUZP2
 145   UNSPEC_VZIP1
 146   UNSPEC_VZIP2
 147   UNSPEC_MISALIGNED_ACCESS
 148   UNSPEC_VCLE
 149   UNSPEC_VCLT
 150 ])
 151
 152
 153 ;; Attribute used to permit string comparisons against <VQH_mnem> in
 154 ;; neon_type attribute definitions.
 155 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
 156
 157 (define_insn "*neon_mov<mode>"
 158   [(set (match_operand:VDX 0 "nonimmediate_operand"
 159           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
 160         (match_operand:VDX 1 "general_operand"
 161           " w,w, Dn,Uni, w, r, r, Usi,r"))]
 162   "TARGET_NEON
 163    && (register_operand (operands[0], <MODE>mode)
 164        || register_operand (operands[1], <MODE>mode))"
 165 {
 166   if (which_alternative == 2)
 167     {
 168       int width, is_valid;
 169       static char templ[40];
 170
 171       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 172         &operands[1], &width);
 173
 174       gcc_assert (is_valid != 0);
 175
 176       if (width == 0)
 177         return "vmov.f32\t%P0, %1  @ <mode>";
 178       else
 179         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
 180
 181       return templ;
 182     }
 183
 184   switch (which_alternative)
 185     {
 186     case 0: return "vmov\t%P0, %P1  @ <mode>";
 187     case 1: case 3: return output_move_neon (operands);
 188     case 2: gcc_unreachable ();
 189     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
 190     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
 191     default: return output_move_double (operands, true, NULL);
 192     }
 193 }
 194  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 195   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
 196   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 197   (set_attr "length" "4,4,4,4,4,4,8,8,8")
 198   (set_attr "pool_range"     "*,*,*,1020,*,*,*,1020,*")
 199   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
 200
 201 (define_insn "*neon_mov<mode>"
 202   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
 203           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
 204         (match_operand:VQXMOV 1 "general_operand"
 205           " w,w, Dn,Uni, w, r, r, Usi, r"))]
 206   "TARGET_NEON
 207    && (register_operand (operands[0], <MODE>mode)
 208        || register_operand (operands[1], <MODE>mode))"
 209 {
 210   if (which_alternative == 2)
 211     {
 212       int width, is_valid;
 213       static char templ[40];
 214
 215       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 216         &operands[1], &width);
 217
 218       gcc_assert (is_valid != 0);
 219
 220       if (width == 0)
 221         return "vmov.f32\t%q0, %1  @ <mode>";
 222       else
 223         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
 224
 225       return templ;
 226     }
 227
 228   switch (which_alternative)
 229     {
 230     case 0: return "vmov\t%q0, %q1  @ <mode>";
 231     case 1: case 3: return output_move_neon (operands);
 232     case 2: gcc_unreachable ();
 233     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 234     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 235     default: return output_move_quad (operands);
 236     }
 237 }
 238   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 239                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 240    (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
 241    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 242    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 243    (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
 244    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 245
 246 (define_expand "movti"
 247   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 248         (match_operand:TI 1 "general_operand" ""))]
 249   "TARGET_NEON"
 250 {
 251   if (can_create_pseudo_p ())
 252     {
 253       if (!REG_P (operands[0]))
 254         operands[1] = force_reg (TImode, operands[1]);
 255     }
 256 })
 257
 258 (define_expand "mov<mode>"
 259   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 260         (match_operand:VSTRUCT 1 "general_operand" ""))]
 261   "TARGET_NEON"
 262 {
 263   if (can_create_pseudo_p ())
 264     {
 265       if (!REG_P (operands[0]))
 266         operands[1] = force_reg (<MODE>mode, operands[1]);
 267     }
 268 })
 269
 270 (define_insn "*neon_mov<mode>"
 271   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 272         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 273   "TARGET_NEON
 274    && (register_operand (operands[0], <MODE>mode)
 275        || register_operand (operands[1], <MODE>mode))"
 276 {
 277   switch (which_alternative)
 278     {
 279     case 0: return "#";
 280     case 1: case 2: return output_move_neon (operands);
 281     default: gcc_unreachable ();
 282     }
 283 }
 284   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 285    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 286
 287 (define_split
 288   [(set (match_operand:EI 0 "s_register_operand" "")
 289         (match_operand:EI 1 "s_register_operand" ""))]
 290   "TARGET_NEON && reload_completed"
 291   [(set (match_dup 0) (match_dup 1))
 292    (set (match_dup 2) (match_dup 3))]
 293 {
 294   int rdest = REGNO (operands[0]);
 295   int rsrc = REGNO (operands[1]);
 296   rtx dest[2], src[2];
 297
 298   dest[0] = gen_rtx_REG (TImode, rdest);
 299   src[0] = gen_rtx_REG (TImode, rsrc);
 300   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 301   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 302
 303   neon_disambiguate_copy (operands, dest, src, 2);
 304 })
 305
 306 (define_split
 307   [(set (match_operand:OI 0 "s_register_operand" "")
 308         (match_operand:OI 1 "s_register_operand" ""))]
 309   "TARGET_NEON && reload_completed"
 310   [(set (match_dup 0) (match_dup 1))
 311    (set (match_dup 2) (match_dup 3))]
 312 {
 313   int rdest = REGNO (operands[0]);
 314   int rsrc = REGNO (operands[1]);
 315   rtx dest[2], src[2];
 316
 317   dest[0] = gen_rtx_REG (TImode, rdest);
 318   src[0] = gen_rtx_REG (TImode, rsrc);
 319   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 320   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 321
 322   neon_disambiguate_copy (operands, dest, src, 2);
 323 })
 324
 325 (define_split
 326   [(set (match_operand:CI 0 "s_register_operand" "")
 327         (match_operand:CI 1 "s_register_operand" ""))]
 328   "TARGET_NEON && reload_completed"
 329   [(set (match_dup 0) (match_dup 1))
 330    (set (match_dup 2) (match_dup 3))
 331    (set (match_dup 4) (match_dup 5))]
 332 {
 333   int rdest = REGNO (operands[0]);
 334   int rsrc = REGNO (operands[1]);
 335   rtx dest[3], src[3];
 336
 337   dest[0] = gen_rtx_REG (TImode, rdest);
 338   src[0] = gen_rtx_REG (TImode, rsrc);
 339   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 340   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 341   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 342   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 343
 344   neon_disambiguate_copy (operands, dest, src, 3);
 345 })
 346
 347 (define_split
 348   [(set (match_operand:XI 0 "s_register_operand" "")
 349         (match_operand:XI 1 "s_register_operand" ""))]
 350   "TARGET_NEON && reload_completed"
 351   [(set (match_dup 0) (match_dup 1))
 352    (set (match_dup 2) (match_dup 3))
 353    (set (match_dup 4) (match_dup 5))
 354    (set (match_dup 6) (match_dup 7))]
 355 {
 356   int rdest = REGNO (operands[0]);
 357   int rsrc = REGNO (operands[1]);
 358   rtx dest[4], src[4];
 359
 360   dest[0] = gen_rtx_REG (TImode, rdest);
 361   src[0] = gen_rtx_REG (TImode, rsrc);
 362   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 363   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 364   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 365   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 366   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 367   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 368
 369   neon_disambiguate_copy (operands, dest, src, 4);
 370 })
 371
 372 (define_expand "movmisalign<mode>"
 373   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 374         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 375                      UNSPEC_MISALIGNED_ACCESS))]
 376   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 377 {
 378   /* This pattern is not permitted to fail during expansion: if both arguments
 379      are non-registers (e.g. memory := constant, which can be created by the
 380      auto-vectorizer), force operand 1 into a register.  */
 381   if (!s_register_operand (operands[0], <MODE>mode)
 382       && !s_register_operand (operands[1], <MODE>mode))
 383     operands[1] = force_reg (<MODE>mode, operands[1]);
 384 })
 385
 386 (define_insn "*movmisalign<mode>_neon_store"
 387   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 388         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 389                     UNSPEC_MISALIGNED_ACCESS))]
 390   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 391   "vst1.<V_sz_elem>\t{%P1}, %A0"
 392   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 393
 394 (define_insn "*movmisalign<mode>_neon_load"
 395   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 396         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 397                     UNSPEC_MISALIGNED_ACCESS))]
 398   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 399   "vld1.<V_sz_elem>\t{%P0}, %A1"
 400   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 401
 402 (define_insn "*movmisalign<mode>_neon_store"
 403   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 404         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 405                     UNSPEC_MISALIGNED_ACCESS))]
 406   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 407   "vst1.<V_sz_elem>\t{%q1}, %A0"
 408   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 409
 410 (define_insn "*movmisalign<mode>_neon_load"
 411   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 412         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 413                     UNSPEC_MISALIGNED_ACCESS))]
 414   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 415   "vld1.<V_sz_elem>\t{%q0}, %A1"
 416   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 417
 418 (define_insn "vec_set<mode>_internal"
 419   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 420         (vec_merge:VD
 421           (vec_duplicate:VD
 422             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 423           (match_operand:VD 3 "s_register_operand" "0,0")
 424           (match_operand:SI 2 "immediate_operand" "i,i")))]
 425   "TARGET_NEON"
 426 {
 427   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 428   if (BYTES_BIG_ENDIAN)
 429     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 430   operands[2] = GEN_INT (elt);
 431
 432   if (which_alternative == 0)
 433     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 434   else
 435     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 436 }
 437   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")])
 438
 439 (define_insn "vec_set<mode>_internal"
 440   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 441         (vec_merge:VQ
 442           (vec_duplicate:VQ
 443             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 444           (match_operand:VQ 3 "s_register_operand" "0,0")
 445           (match_operand:SI 2 "immediate_operand" "i,i")))]
 446   "TARGET_NEON"
 447 {
 448   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 449   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 450   int elt = elem % half_elts;
 451   int hi = (elem / half_elts) * 2;
 452   int regno = REGNO (operands[0]);
 453
 454   if (BYTES_BIG_ENDIAN)
 455     elt = half_elts - 1 - elt;
 456
 457   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 458   operands[2] = GEN_INT (elt);
 459
 460   if (which_alternative == 0)
 461     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 462   else
 463     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 464 }
 465   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
 466 )
 467
 468 (define_insn "vec_setv2di_internal"
 469   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 470         (vec_merge:V2DI
 471           (vec_duplicate:V2DI
 472             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 473           (match_operand:V2DI 3 "s_register_operand" "0,0")
 474           (match_operand:SI 2 "immediate_operand" "i,i")))]
 475   "TARGET_NEON"
 476 {
 477   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 478   int regno = REGNO (operands[0]) + 2 * elem;
 479
 480   operands[0] = gen_rtx_REG (DImode, regno);
 481
 482   if (which_alternative == 0)
 483     return "vld1.64\t%P0, %A1";
 484   else
 485     return "vmov\t%P0, %Q1, %R1";
 486 }
 487   [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")]
 488 )
 489
 490 (define_expand "vec_set<mode>"
 491   [(match_operand:VDQ 0 "s_register_operand" "")
 492    (match_operand:<V_elem> 1 "s_register_operand" "")
 493    (match_operand:SI 2 "immediate_operand" "")]
 494   "TARGET_NEON"
 495 {
 496   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 497   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 498                                          GEN_INT (elem), operands[0]));
 499   DONE;
 500 })
 501
 502 (define_insn "vec_extract<mode>"
 503   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 504         (vec_select:<V_elem>
 505           (match_operand:VD 1 "s_register_operand" "w,w")
 506           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 507   "TARGET_NEON"
 508 {
 509   if (BYTES_BIG_ENDIAN)
 510     {
 511       int elt = INTVAL (operands[2]);
 512       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 513       operands[2] = GEN_INT (elt);
 514     }
 515
 516   if (which_alternative == 0)
 517     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 518   else
 519     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 520 }
 521   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 522 )
 523
 524 (define_insn "vec_extract<mode>"
 525   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 526         (vec_select:<V_elem>
 527           (match_operand:VQ 1 "s_register_operand" "w,w")
 528           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 529   "TARGET_NEON"
 530 {
 531   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 532   int elt = INTVAL (operands[2]) % half_elts;
 533   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 534   int regno = REGNO (operands[1]);
 535
 536   if (BYTES_BIG_ENDIAN)
 537     elt = half_elts - 1 - elt;
 538
 539   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 540   operands[2] = GEN_INT (elt);
 541
 542   if (which_alternative == 0)
 543     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 544   else
 545     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 546 }
 547   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 548 )
 549
 550 (define_insn "vec_extractv2di"
 551   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 552         (vec_select:DI
 553           (match_operand:V2DI 1 "s_register_operand" "w,w")
 554           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 555   "TARGET_NEON"
 556 {
 557   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 558
 559   operands[1] = gen_rtx_REG (DImode, regno);
 560
 561   if (which_alternative == 0)
 562     return "vst1.64\t{%P1}, %A0  @ v2di";
 563   else
 564     return "vmov\t%Q0, %R0, %P1  @ v2di";
 565 }
 566   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")]
 567 )
 568
 569 (define_expand "vec_init<mode>"
 570   [(match_operand:VDQ 0 "s_register_operand" "")
 571    (match_operand 1 "" "")]
 572   "TARGET_NEON"
 573 {
 574   neon_expand_vector_init (operands[0], operands[1]);
 575   DONE;
 576 })
 577
 578 ;; Doubleword and quadword arithmetic.
 579
 580 ;; NOTE: some other instructions also support 64-bit integer
 581 ;; element size, which we could potentially use for "long long" operations.
 582
 583 (define_insn "*add<mode>3_neon"
 584   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 585         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 586                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 587   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 588   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 589   [(set (attr "neon_type")
 590       (if_then_else (match_test "<Is_float_mode>")
 591                     (if_then_else (match_test "<Is_d_reg>")
 592                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 593                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 594                     (const_string "neon_int_1")))]
 595 )
 596
 597 (define_insn "adddi3_neon"
 598   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 599         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 600                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 601    (clobber (reg:CC CC_REGNUM))]
 602   "TARGET_NEON"
 603 {
 604   switch (which_alternative)
 605     {
 606     case 0: /* fall through */
 607     case 3: return "vadd.i64\t%P0, %P1, %P2";
 608     case 1: return "#";
 609     case 2: return "#";
 610     case 4: return "#";
 611     case 5: return "#";
 612     case 6: return "#";
 613     default: gcc_unreachable ();
 614     }
 615 }
 616   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 617    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 618    (set_attr "length" "*,8,8,*,8,8,8")
 619    (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
 620 )
 621
 622 (define_insn "*sub<mode>3_neon"
 623   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 624         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 625                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 626   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 627   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 628   [(set (attr "neon_type")
 629       (if_then_else (match_test "<Is_float_mode>")
 630                     (if_then_else (match_test "<Is_d_reg>")
 631                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 632                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 633                     (const_string "neon_int_2")))]
 634 )
 635
 636 (define_insn "subdi3_neon"
 637   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 638         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 639                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 640    (clobber (reg:CC CC_REGNUM))]
 641   "TARGET_NEON"
 642 {
 643   switch (which_alternative)
 644     {
 645     case 0: /* fall through */
 646     case 4: return "vsub.i64\t%P0, %P1, %P2";
 647     case 1: /* fall through */
 648     case 2: /* fall through */
 649     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 650     default: gcc_unreachable ();
 651     }
 652 }
 653   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 654    (set_attr "conds" "*,clob,clob,clob,*")
 655    (set_attr "length" "*,8,8,8,*")
 656    (set_attr "arch" "nota8,*,*,*,onlya8")]
 657 )
 658
 659 (define_insn "*mul<mode>3_neon"
 660   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 661         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 662                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 663   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 664   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 665   [(set (attr "neon_type")
 666       (if_then_else (match_test "<Is_float_mode>")
 667                     (if_then_else (match_test "<Is_d_reg>")
 668                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 669                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 670                     (if_then_else (match_test "<Is_d_reg>")
 671                                   (if_then_else
 672                                     (match_test "<Scalar_mul_8_16>")
 673                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 674                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 675                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 676                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 677                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 678 )
 679
 680 (define_insn "mul<mode>3add<mode>_neon"
 681   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 682         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 683                             (match_operand:VDQ 3 "s_register_operand" "w"))
 684                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 685   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 686   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 687   [(set (attr "neon_type")
 688       (if_then_else (match_test "<Is_float_mode>")
 689                     (if_then_else (match_test "<Is_d_reg>")
 690                                   (const_string "neon_fp_vmla_ddd")
 691                                   (const_string "neon_fp_vmla_qqq"))
 692                     (if_then_else (match_test "<Is_d_reg>")
 693                                   (if_then_else
 694                                     (match_test "<Scalar_mul_8_16>")
 695                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 696                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 697                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 698                                     (const_string "neon_mla_qqq_8_16")
 699                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 700 )
 701
 702 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 703   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 704         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 705                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 706                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 707   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 708   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 709   [(set (attr "neon_type")
 710       (if_then_else (match_test "<Is_float_mode>")
 711                     (if_then_else (match_test "<Is_d_reg>")
 712                                   (const_string "neon_fp_vmla_ddd")
 713                                   (const_string "neon_fp_vmla_qqq"))
 714                     (if_then_else (match_test "<Is_d_reg>")
 715                                   (if_then_else
 716                                     (match_test "<Scalar_mul_8_16>")
 717                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 718                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 719                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 720                                     (const_string "neon_mla_qqq_8_16")
 721                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 722 )
 723
 724 ;; Fused multiply-accumulate
 725 (define_insn "fma<VCVTF:mode>4"
 726   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 727         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 728                  (match_operand:VCVTF 2 "register_operand" "w")
 729                  (match_operand:VCVTF 3 "register_operand" "0")))]
 730   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 731   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 732   [(set (attr "neon_type")
 733         (if_then_else (match_test "<Is_d_reg>")
 734                       (const_string "neon_fp_vmla_ddd")
 735                       (const_string "neon_fp_vmla_qqq")))]
 736 )
 737
 738 (define_insn "*fmsub<VCVTF:mode>4"
 739   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 740         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 741                    (match_operand:VCVTF 2 "register_operand" "w")
 742                    (match_operand:VCVTF 3 "register_operand" "0")))]
 743   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 744   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 745   [(set (attr "neon_type")
 746         (if_then_else (match_test "<Is_d_reg>")
 747                       (const_string "neon_fp_vmla_ddd")
 748                       (const_string "neon_fp_vmla_qqq")))]
 749 )
 750
 751 (define_insn "ior<mode>3"
 752   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 753         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 754                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 755   "TARGET_NEON"
 756 {
 757   switch (which_alternative)
 758     {
 759     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 760     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 761                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 762     default: gcc_unreachable ();
 763     }
 764 }
 765   [(set_attr "neon_type" "neon_int_1")]
 766 )
 767
 768 (define_insn "iordi3_neon"
 769   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 770         (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 771                 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
 772   "TARGET_NEON"
 773 {
 774   switch (which_alternative)
 775     {
 776     case 0: /* fall through */
 777     case 4: return "vorr\t%P0, %P1, %P2";
 778     case 1: /* fall through */
 779     case 5: return neon_output_logic_immediate ("vorr", &operands[2],
 780                      DImode, 0, VALID_NEON_QREG_MODE (DImode));
 781     case 2: return "#";
 782     case 3: return "#";
 783     default: gcc_unreachable ();
 784     }
 785 }
 786   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 787    (set_attr "length" "*,*,8,8,*,*")
 788    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 789 )
 790
 791 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 792 ;; vorr. We support the pseudo-instruction vand instead, because that
 793 ;; corresponds to the canonical form the middle-end expects to use for
 794 ;; immediate bitwise-ANDs.
 795
 796 (define_insn "and<mode>3"
 797   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 798         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 799                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 800   "TARGET_NEON"
 801 {
 802   switch (which_alternative)
 803     {
 804     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 805     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 806                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 807     default: gcc_unreachable ();
 808     }
 809 }
 810   [(set_attr "neon_type" "neon_int_1")]
 811 )
 812
 813 (define_insn "anddi3_neon"
 814   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 815         (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 816                 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
 817   "TARGET_NEON"
 818 {
 819   switch (which_alternative)
 820     {
 821     case 0: /* fall through */
 822     case 4: return "vand\t%P0, %P1, %P2";
 823     case 1: /* fall through */
 824     case 5: return neon_output_logic_immediate ("vand", &operands[2],
 825                      DImode, 1, VALID_NEON_QREG_MODE (DImode));
 826     case 2: return "#";
 827     case 3: return "#";
 828     default: gcc_unreachable ();
 829     }
 830 }
 831   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 832    (set_attr "length" "*,*,8,8,*,*")
 833    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 834 )
 835
 836 (define_insn "orn<mode>3_neon"
 837   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 838         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 839                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 840   "TARGET_NEON"
 841   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 842   [(set_attr "neon_type" "neon_int_1")]
 843 )
 844
 845 ;; TODO: investigate whether we should disable
 846 ;; this and bicdi3_neon for the A8 in line with the other
 847 ;; changes above.
 848 (define_insn_and_split "orndi3_neon"
 849   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 850         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 851                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 852   "TARGET_NEON"
 853   "@
 854    vorn\t%P0, %P1, %P2
 855    #
 856    #
 857    #"
 858   "reload_completed &&
 859    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 860   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 861    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 862   "
 863   {
 864     if (TARGET_THUMB2)
 865       {
 866         operands[3] = gen_highpart (SImode, operands[0]);
 867         operands[0] = gen_lowpart (SImode, operands[0]);
 868         operands[4] = gen_highpart (SImode, operands[2]);
 869         operands[2] = gen_lowpart (SImode, operands[2]);
 870         operands[5] = gen_highpart (SImode, operands[1]);
 871         operands[1] = gen_lowpart (SImode, operands[1]);
 872       }
 873     else
 874       {
 875         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 876         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 877         DONE;
 878       }
 879   }"
 880   [(set_attr "neon_type" "neon_int_1,*,*,*")
 881    (set_attr "length" "*,16,8,8")
 882    (set_attr "arch" "any,a,t2,t2")]
 883 )
 884
 885 (define_insn "bic<mode>3_neon"
 886   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 887         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 888                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 889   "TARGET_NEON"
 890   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 891   [(set_attr "neon_type" "neon_int_1")]
 892 )
 893
 894 ;; Compare to *anddi_notdi_di.
 895 (define_insn "bicdi3_neon"
 896   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 897         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 898                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 899   "TARGET_NEON"
 900   "@
 901    vbic\t%P0, %P1, %P2
 902    #
 903    #"
 904   [(set_attr "neon_type" "neon_int_1,*,*")
 905    (set_attr "length" "*,8,8")]
 906 )
 907
 908 (define_insn "xor<mode>3"
 909   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 910         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 911                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 912   "TARGET_NEON"
 913   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 914   [(set_attr "neon_type" "neon_int_1")]
 915 )
 916
 917 (define_insn "xordi3_neon"
 918   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
 919         (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
 920                 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
 921   "TARGET_NEON"
 922   "@
 923    veor\t%P0, %P1, %P2
 924    #
 925    #
 926    veor\t%P0, %P1, %P2"
 927   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
 928    (set_attr "length" "*,8,8,*")
 929    (set_attr "arch" "nota8,*,*,onlya8")]
 930 )
 931
 932 (define_insn "one_cmpl<mode>2"
 933   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 934         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 935   "TARGET_NEON"
 936   "vmvn\t%<V_reg>0, %<V_reg>1"
 937   [(set_attr "neon_type" "neon_int_1")]
 938 )
 939
 940 (define_insn "abs<mode>2"
 941   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 942         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 943   "TARGET_NEON"
 944   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 945   [(set (attr "neon_type")
 946       (if_then_else (match_test "<Is_float_mode>")
 947                     (if_then_else (match_test "<Is_d_reg>")
 948                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 949                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 950                     (const_string "neon_int_3")))]
 951 )
 952
 953 (define_insn "neg<mode>2"
 954   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 955         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 956   "TARGET_NEON"
 957   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 958   [(set (attr "neon_type")
 959       (if_then_else (match_test "<Is_float_mode>")
 960                     (if_then_else (match_test "<Is_d_reg>")
 961                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 962                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 963                     (const_string "neon_int_3")))]
 964 )
 965
 966 (define_insn "negdi2_neon"
 967   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 968         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 969    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 970    (clobber (reg:CC CC_REGNUM))]
 971   "TARGET_NEON"
 972   "#"
 973   [(set_attr "length" "8")]
 974 )
 975
 976 ; Split negdi2_neon for vfp registers
 977 (define_split
 978   [(set (match_operand:DI 0 "s_register_operand" "")
 979         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 980    (clobber (match_scratch:DI 2 ""))
 981    (clobber (reg:CC CC_REGNUM))]
 982   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 983   [(set (match_dup 2) (const_int 0))
 984    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 985               (clobber (reg:CC CC_REGNUM))])]
 986   {
 987     if (!REG_P (operands[2]))
 988       operands[2] = operands[0];
 989   }
 990 )
 991
 992 ; Split negdi2_neon for core registers
 993 (define_split
 994   [(set (match_operand:DI 0 "s_register_operand" "")
 995         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 996    (clobber (match_scratch:DI 2 ""))
 997    (clobber (reg:CC CC_REGNUM))]
 998   "TARGET_32BIT && reload_completed
 999    && arm_general_register_operand (operands[0], DImode)"
1000   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
1001               (clobber (reg:CC CC_REGNUM))])]
1002   ""
1003 )
1004
1005 (define_insn "*umin<mode>3_neon"
1006   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1007         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1008                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
1009   "TARGET_NEON"
1010   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1011   [(set_attr "neon_type" "neon_int_5")]
1012 )
1013
1014 (define_insn "*umax<mode>3_neon"
1015   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1016         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1017                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
1018   "TARGET_NEON"
1019   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1020   [(set_attr "neon_type" "neon_int_5")]
1021 )
1022
1023 (define_insn "*smin<mode>3_neon"
1024   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1025         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
1026                    (match_operand:VDQW 2 "s_register_operand" "w")))]
1027   "TARGET_NEON"
1028   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1029   [(set (attr "neon_type")
1030       (if_then_else (match_test "<Is_float_mode>")
1031                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1032                     (const_string "neon_int_5")))]
1033 )
1034
1035 (define_insn "*smax<mode>3_neon"
1036   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1037         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
1038                    (match_operand:VDQW 2 "s_register_operand" "w")))]
1039   "TARGET_NEON"
1040   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1041   [(set (attr "neon_type")
1042       (if_then_else (match_test "<Is_float_mode>")
1043                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1044                     (const_string "neon_int_5")))]
1045 )
1046
1047 ; TODO: V2DI shifts are current disabled because there are bugs in the
1048 ; generic vectorizer code.  It ends up creating a V2DI constructor with
1049 ; SImode elements.
1050
1051 (define_insn "vashl<mode>3"
1052   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
1053         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
1054                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
1055   "TARGET_NEON"
1056   {
1057     switch (which_alternative)
1058       {
1059         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
1060         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
1061                                                     <MODE>mode,
1062                                                     VALID_NEON_QREG_MODE (<MODE>mode),
1063                                                     true);
1064         default: gcc_unreachable ();
1065       }
1066   }
1067   [(set (attr "neon_type")
1068       (if_then_else (match_test "<Is_d_reg>")
1069                     (const_string "neon_vshl_ddd")
1070                     (const_string "neon_shift_3")))]
1071 )
1072
1073 (define_insn "vashr<mode>3_imm"
1074   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1075         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1076                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1077   "TARGET_NEON"
1078   {
1079     return neon_output_shift_immediate ("vshr", 's', &operands[2],
1080                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1081                                         false);
1082   }
1083   [(set (attr "neon_type")
1084       (if_then_else (match_test "<Is_d_reg>")
1085                     (const_string "neon_vshl_ddd")
1086                     (const_string "neon_shift_3")))]
1087 )
1088
1089 (define_insn "vlshr<mode>3_imm"
1090   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1091         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1092                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1093   "TARGET_NEON"
1094   {
1095     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1096                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1097                                         false);
1098   }
1099   [(set (attr "neon_type")
1100         (if_then_else (match_test "<Is_d_reg>")
1101                       (const_string "neon_vshl_ddd")
1102                       (const_string "neon_shift_3")))]
1103 )
1104
1105 ; Used for implementing logical shift-right, which is a left-shift by a negative
1106 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1107 ; above, but using an unspec in case GCC tries anything tricky with negative
1108 ; shift amounts.
1109
1110 (define_insn "ashl<mode>3_signed"
1111   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1112         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1113                       (match_operand:VDQI 2 "s_register_operand" "w")]
1114                      UNSPEC_ASHIFT_SIGNED))]
1115   "TARGET_NEON"
1116   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1117   [(set (attr "neon_type")
1118       (if_then_else (match_test "<Is_d_reg>")
1119                     (const_string "neon_vshl_ddd")
1120                     (const_string "neon_shift_3")))]
1121 )
1122
1123 ; Used for implementing logical shift-right, which is a left-shift by a negative
1124 ; amount, with unsigned operands.
1125
1126 (define_insn "ashl<mode>3_unsigned"
1127   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1128         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1129                       (match_operand:VDQI 2 "s_register_operand" "w")]
1130                      UNSPEC_ASHIFT_UNSIGNED))]
1131   "TARGET_NEON"
1132   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1133   [(set (attr "neon_type")
1134       (if_then_else (match_test "<Is_d_reg>")
1135                     (const_string "neon_vshl_ddd")
1136                     (const_string "neon_shift_3")))]
1137 )
1138
1139 (define_expand "vashr<mode>3"
1140   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1141         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1142                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1143   "TARGET_NEON"
1144 {
1145   if (s_register_operand (operands[2], <MODE>mode))
1146     {
1147       rtx neg = gen_reg_rtx (<MODE>mode);
1148       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1149       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1150     }
1151   else
1152     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1153   DONE;
1154 })
1155
1156 (define_expand "vlshr<mode>3"
1157   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1158         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1159                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1160   "TARGET_NEON"
1161 {
1162   if (s_register_operand (operands[2], <MODE>mode))
1163     {
1164       rtx neg = gen_reg_rtx (<MODE>mode);
1165       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1166       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1167     }
1168   else
1169     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1170   DONE;
1171 })
1172
1173 ;; Widening operations
1174
1175 (define_insn "widen_ssum<mode>3"
1176   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1177         (plus:<V_widen> (sign_extend:<V_widen>
1178                           (match_operand:VW 1 "s_register_operand" "%w"))
1179                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1180   "TARGET_NEON"
1181   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1182   [(set_attr "neon_type" "neon_int_3")]
1183 )
1184
1185 (define_insn "widen_usum<mode>3"
1186   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1187         (plus:<V_widen> (zero_extend:<V_widen>
1188                           (match_operand:VW 1 "s_register_operand" "%w"))
1189                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1190   "TARGET_NEON"
1191   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1192   [(set_attr "neon_type" "neon_int_3")]
1193 )
1194
1195 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1196 ;; shift-count granularity. That's good enough for the middle-end's current
1197 ;; needs.
1198
1199 ;; Note that it's not safe to perform such an operation in big-endian mode,
1200 ;; due to element-ordering issues.
1201
1202 (define_expand "vec_shr_<mode>"
1203   [(match_operand:VDQ 0 "s_register_operand" "")
1204    (match_operand:VDQ 1 "s_register_operand" "")
1205    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1206   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1207 {
1208   rtx zero_reg;
1209   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1210   const int width = GET_MODE_BITSIZE (<MODE>mode);
1211   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1212   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1213     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1214
1215   if (num_bits == width)
1216     {
1217       emit_move_insn (operands[0], operands[1]);
1218       DONE;
1219     }
1220
1221   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1222   operands[0] = gen_lowpart (bvecmode, operands[0]);
1223   operands[1] = gen_lowpart (bvecmode, operands[1]);
1224
1225   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1226                       GEN_INT (num_bits / BITS_PER_UNIT)));
1227   DONE;
1228 })
1229
1230 (define_expand "vec_shl_<mode>"
1231   [(match_operand:VDQ 0 "s_register_operand" "")
1232    (match_operand:VDQ 1 "s_register_operand" "")
1233    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1234   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1235 {
1236   rtx zero_reg;
1237   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1238   const int width = GET_MODE_BITSIZE (<MODE>mode);
1239   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1240   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1241     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1242
1243   if (num_bits == 0)
1244     {
1245       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1246       DONE;
1247     }
1248
1249   num_bits = width - num_bits;
1250
1251   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1252   operands[0] = gen_lowpart (bvecmode, operands[0]);
1253   operands[1] = gen_lowpart (bvecmode, operands[1]);
1254
1255   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1256                       GEN_INT (num_bits / BITS_PER_UNIT)));
1257   DONE;
1258 })
1259
1260 ;; Helpers for quad-word reduction operations
1261
1262 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1263 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1264 ; N/2-element vector.
1265
1266 (define_insn "quad_halves_<code>v4si"
1267   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1268         (vqh_ops:V2SI
1269           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1270                            (parallel [(const_int 0) (const_int 1)]))
1271           (vec_select:V2SI (match_dup 1)
1272                            (parallel [(const_int 2) (const_int 3)]))))]
1273   "TARGET_NEON"
1274   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1275   [(set_attr "vqh_mnem" "<VQH_mnem>")
1276    (set (attr "neon_type")
1277       (if_then_else (eq_attr "vqh_mnem" "vadd")
1278                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1279 )
1280
1281 (define_insn "quad_halves_<code>v4sf"
1282   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1283         (vqhs_ops:V2SF
1284           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1285                            (parallel [(const_int 0) (const_int 1)]))
1286           (vec_select:V2SF (match_dup 1)
1287                            (parallel [(const_int 2) (const_int 3)]))))]
1288   "TARGET_NEON && flag_unsafe_math_optimizations"
1289   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1290   [(set_attr "vqh_mnem" "<VQH_mnem>")
1291    (set (attr "neon_type")
1292       (if_then_else (eq_attr "vqh_mnem" "vadd")
1293                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1294 )
1295
1296 (define_insn "quad_halves_<code>v8hi"
1297   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1298         (vqh_ops:V4HI
1299           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1300                            (parallel [(const_int 0) (const_int 1)
1301                                       (const_int 2) (const_int 3)]))
1302           (vec_select:V4HI (match_dup 1)
1303                            (parallel [(const_int 4) (const_int 5)
1304                                       (const_int 6) (const_int 7)]))))]
1305   "TARGET_NEON"
1306   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1307   [(set_attr "vqh_mnem" "<VQH_mnem>")
1308    (set (attr "neon_type")
1309       (if_then_else (eq_attr "vqh_mnem" "vadd")
1310                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1311 )
1312
1313 (define_insn "quad_halves_<code>v16qi"
1314   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1315         (vqh_ops:V8QI
1316           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1317                            (parallel [(const_int 0) (const_int 1)
1318                                       (const_int 2) (const_int 3)
1319                                       (const_int 4) (const_int 5)
1320                                       (const_int 6) (const_int 7)]))
1321           (vec_select:V8QI (match_dup 1)
1322                            (parallel [(const_int 8) (const_int 9)
1323                                       (const_int 10) (const_int 11)
1324                                       (const_int 12) (const_int 13)
1325                                       (const_int 14) (const_int 15)]))))]
1326   "TARGET_NEON"
1327   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1328   [(set_attr "vqh_mnem" "<VQH_mnem>")
1329    (set (attr "neon_type")
1330       (if_then_else (eq_attr "vqh_mnem" "vadd")
1331                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1332 )
1333
1334 (define_expand "move_hi_quad_<mode>"
1335  [(match_operand:ANY128 0 "s_register_operand" "")
1336   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1337  "TARGET_NEON"
1338 {
1339   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1340                                        GET_MODE_SIZE (<V_HALF>mode)),
1341                   operands[1]);
1342   DONE;
1343 })
1344
1345 (define_expand "move_lo_quad_<mode>"
1346  [(match_operand:ANY128 0 "s_register_operand" "")
1347   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1348  "TARGET_NEON"
1349 {
1350   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1351                                        <MODE>mode, 0),
1352                   operands[1]);
1353   DONE;
1354 })
1355
1356 ;; Reduction operations
1357
1358 (define_expand "reduc_splus_<mode>"
1359   [(match_operand:VD 0 "s_register_operand" "")
1360    (match_operand:VD 1 "s_register_operand" "")]
1361   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1362 {
1363   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1364                         &gen_neon_vpadd_internal<mode>);
1365   DONE;
1366 })
1367
1368 (define_expand "reduc_splus_<mode>"
1369   [(match_operand:VQ 0 "s_register_operand" "")
1370    (match_operand:VQ 1 "s_register_operand" "")]
1371   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1372    && !BYTES_BIG_ENDIAN"
1373 {
1374   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1375   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1376
1377   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1378   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1379   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1380
1381   DONE;
1382 })
1383
1384 (define_insn "reduc_splus_v2di"
1385   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1386         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1387                      UNSPEC_VPADD))]
1388   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1389   "vadd.i64\t%e0, %e1, %f1"
1390   [(set_attr "neon_type" "neon_int_1")]
1391 )
1392
1393 ;; NEON does not distinguish between signed and unsigned addition except on
1394 ;; widening operations.
1395 (define_expand "reduc_uplus_<mode>"
1396   [(match_operand:VDQI 0 "s_register_operand" "")
1397    (match_operand:VDQI 1 "s_register_operand" "")]
1398   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1399 {
1400   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1401   DONE;
1402 })
1403
1404 (define_expand "reduc_smin_<mode>"
1405   [(match_operand:VD 0 "s_register_operand" "")
1406    (match_operand:VD 1 "s_register_operand" "")]
1407   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1408 {
1409   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1410                         &gen_neon_vpsmin<mode>);
1411   DONE;
1412 })
1413
1414 (define_expand "reduc_smin_<mode>"
1415   [(match_operand:VQ 0 "s_register_operand" "")
1416    (match_operand:VQ 1 "s_register_operand" "")]
1417   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1418    && !BYTES_BIG_ENDIAN"
1419 {
1420   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1421   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1422
1423   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1424   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1425   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1426
1427   DONE;
1428 })
1429
1430 (define_expand "reduc_smax_<mode>"
1431   [(match_operand:VD 0 "s_register_operand" "")
1432    (match_operand:VD 1 "s_register_operand" "")]
1433   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1434 {
1435   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1436                         &gen_neon_vpsmax<mode>);
1437   DONE;
1438 })
1439
1440 (define_expand "reduc_smax_<mode>"
1441   [(match_operand:VQ 0 "s_register_operand" "")
1442    (match_operand:VQ 1 "s_register_operand" "")]
1443   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1444    && !BYTES_BIG_ENDIAN"
1445 {
1446   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1447   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1448
1449   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1450   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1451   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1452
1453   DONE;
1454 })
1455
1456 (define_expand "reduc_umin_<mode>"
1457   [(match_operand:VDI 0 "s_register_operand" "")
1458    (match_operand:VDI 1 "s_register_operand" "")]
1459   "TARGET_NEON"
1460 {
1461   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1462                         &gen_neon_vpumin<mode>);
1463   DONE;
1464 })
1465
1466 (define_expand "reduc_umin_<mode>"
1467   [(match_operand:VQI 0 "s_register_operand" "")
1468    (match_operand:VQI 1 "s_register_operand" "")]
1469   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1470 {
1471   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1472   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1473
1474   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1475   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1476   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1477
1478   DONE;
1479 })
1480
1481 (define_expand "reduc_umax_<mode>"
1482   [(match_operand:VDI 0 "s_register_operand" "")
1483    (match_operand:VDI 1 "s_register_operand" "")]
1484   "TARGET_NEON"
1485 {
1486   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1487                         &gen_neon_vpumax<mode>);
1488   DONE;
1489 })
1490
1491 (define_expand "reduc_umax_<mode>"
1492   [(match_operand:VQI 0 "s_register_operand" "")
1493    (match_operand:VQI 1 "s_register_operand" "")]
1494   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1495 {
1496   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1497   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1498
1499   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1500   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1501   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1502
1503   DONE;
1504 })
1505
1506 (define_insn "neon_vpadd_internal<mode>"
1507   [(set (match_operand:VD 0 "s_register_operand" "=w")
1508         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1509                     (match_operand:VD 2 "s_register_operand" "w")]
1510                    UNSPEC_VPADD))]
1511   "TARGET_NEON"
1512   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1513   ;; Assume this schedules like vadd.
1514   [(set (attr "neon_type")
1515       (if_then_else (match_test "<Is_float_mode>")
1516                     (if_then_else (match_test "<Is_d_reg>")
1517                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1518                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1519                     (const_string "neon_int_1")))]
1520 )
1521
1522 (define_insn "neon_vpsmin<mode>"
1523   [(set (match_operand:VD 0 "s_register_operand" "=w")
1524         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1525                     (match_operand:VD 2 "s_register_operand" "w")]
1526                    UNSPEC_VPSMIN))]
1527   "TARGET_NEON"
1528   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1529   ;; Assume this schedules like vmin.
1530   [(set (attr "neon_type")
1531       (if_then_else (match_test "<Is_float_mode>")
1532                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1533                     (const_string "neon_int_5")))]
1534 )
1535
1536 (define_insn "neon_vpsmax<mode>"
1537   [(set (match_operand:VD 0 "s_register_operand" "=w")
1538         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1539                     (match_operand:VD 2 "s_register_operand" "w")]
1540                    UNSPEC_VPSMAX))]
1541   "TARGET_NEON"
1542   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1543   ;; Assume this schedules like vmax.
1544   [(set (attr "neon_type")
1545       (if_then_else (match_test "<Is_float_mode>")
1546                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1547                     (const_string "neon_int_5")))]
1548 )
1549
1550 (define_insn "neon_vpumin<mode>"
1551   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1552         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1553                      (match_operand:VDI 2 "s_register_operand" "w")]
1554                    UNSPEC_VPUMIN))]
1555   "TARGET_NEON"
1556   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1557   ;; Assume this schedules like umin.
1558   [(set_attr "neon_type" "neon_int_5")]
1559 )
1560
1561 (define_insn "neon_vpumax<mode>"
1562   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1563         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1564                      (match_operand:VDI 2 "s_register_operand" "w")]
1565                    UNSPEC_VPUMAX))]
1566   "TARGET_NEON"
1567   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1568   ;; Assume this schedules like umax.
1569   [(set_attr "neon_type" "neon_int_5")]
1570 )
1571
1572 ;; Saturating arithmetic
1573
1574 ; NOTE: Neon supports many more saturating variants of instructions than the
1575 ; following, but these are all GCC currently understands.
1576 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1577 ; yet either, although these patterns may be used by intrinsics when they're
1578 ; added.
1579
1580 (define_insn "*ss_add<mode>_neon"
1581   [(set (match_operand:VD 0 "s_register_operand" "=w")
1582        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1583                    (match_operand:VD 2 "s_register_operand" "w")))]
1584   "TARGET_NEON"
1585   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1586   [(set_attr "neon_type" "neon_int_4")]
1587 )
1588
1589 (define_insn "*us_add<mode>_neon"
1590   [(set (match_operand:VD 0 "s_register_operand" "=w")
1591        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1592                    (match_operand:VD 2 "s_register_operand" "w")))]
1593   "TARGET_NEON"
1594   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1595   [(set_attr "neon_type" "neon_int_4")]
1596 )
1597
1598 (define_insn "*ss_sub<mode>_neon"
1599   [(set (match_operand:VD 0 "s_register_operand" "=w")
1600        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1601                     (match_operand:VD 2 "s_register_operand" "w")))]
1602   "TARGET_NEON"
1603   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1604   [(set_attr "neon_type" "neon_int_5")]
1605 )
1606
1607 (define_insn "*us_sub<mode>_neon"
1608   [(set (match_operand:VD 0 "s_register_operand" "=w")
1609        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1610                     (match_operand:VD 2 "s_register_operand" "w")))]
1611   "TARGET_NEON"
1612   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1613   [(set_attr "neon_type" "neon_int_5")]
1614 )
1615
1616 ;; Conditional instructions.  These are comparisons with conditional moves for
1617 ;; vectors.  They perform the assignment:
1618 ;;
1619 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1620 ;;
1621 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1622 ;; element-wise.
1623
1624 (define_expand "vcond<mode><mode>"
1625   [(set (match_operand:VDQW 0 "s_register_operand" "")
1626         (if_then_else:VDQW
1627           (match_operator 3 "arm_comparison_operator"
1628             [(match_operand:VDQW 4 "s_register_operand" "")
1629              (match_operand:VDQW 5 "nonmemory_operand" "")])
1630           (match_operand:VDQW 1 "s_register_operand" "")
1631           (match_operand:VDQW 2 "s_register_operand" "")))]
1632   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1633 {
1634   rtx mask;
1635   int inverse = 0, immediate_zero = 0;
1636   /* See the description of "magic" bits in the 'T' case of
1637      arm_print_operand.  */
1638   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1639                              ? 3 : 1;
1640   rtx magic_rtx = GEN_INT (magic_word);
1641
1642   mask = gen_reg_rtx (<V_cmp_result>mode);
1643
1644   if (operands[5] == CONST0_RTX (<MODE>mode))
1645     immediate_zero = 1;
1646   else if (!REG_P (operands[5]))
1647     operands[5] = force_reg (<MODE>mode, operands[5]);
1648
1649   switch (GET_CODE (operands[3]))
1650     {
1651     case GE:
1652       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1653                                       magic_rtx));
1654       break;
1655
1656     case GT:
1657       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1658                                       magic_rtx));
1659       break;
1660
1661     case EQ:
1662       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1663                                       magic_rtx));
1664       break;
1665
1666     case LE:
1667       if (immediate_zero)
1668         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1669                                         magic_rtx));
1670       else
1671         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1672                                         magic_rtx));
1673       break;
1674
1675     case LT:
1676       if (immediate_zero)
1677         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1678                                         magic_rtx));
1679       else
1680         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1681                                         magic_rtx));
1682       break;
1683
1684     case NE:
1685       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1686                                       magic_rtx));
1687       inverse = 1;
1688       break;
1689
1690     default:
1691       gcc_unreachable ();
1692     }
1693
1694   if (inverse)
1695     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1696                                     operands[1]));
1697   else
1698     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1699                                     operands[2]));
1700
1701   DONE;
1702 })
1703
1704 (define_expand "vcondu<mode><mode>"
1705   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1706         (if_then_else:VDQIW
1707           (match_operator 3 "arm_comparison_operator"
1708             [(match_operand:VDQIW 4 "s_register_operand" "")
1709              (match_operand:VDQIW 5 "s_register_operand" "")])
1710           (match_operand:VDQIW 1 "s_register_operand" "")
1711           (match_operand:VDQIW 2 "s_register_operand" "")))]
1712   "TARGET_NEON"
1713 {
1714   rtx mask;
1715   int inverse = 0, immediate_zero = 0;
1716
1717   mask = gen_reg_rtx (<V_cmp_result>mode);
1718
1719   if (operands[5] == CONST0_RTX (<MODE>mode))
1720     immediate_zero = 1;
1721   else if (!REG_P (operands[5]))
1722     operands[5] = force_reg (<MODE>mode, operands[5]);
1723
1724   switch (GET_CODE (operands[3]))
1725     {
1726     case GEU:
1727       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1728                                       const0_rtx));
1729       break;
1730
1731     case GTU:
1732       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1733                                       const0_rtx));
1734       break;
1735
1736     case EQ:
1737       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1738                                       const0_rtx));
1739       break;
1740
1741     case LEU:
1742       if (immediate_zero)
1743         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1744                                         const0_rtx));
1745       else
1746         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1747                                         const0_rtx));
1748       break;
1749
1750     case LTU:
1751       if (immediate_zero)
1752         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1753                                         const0_rtx));
1754       else
1755         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1756                                         const0_rtx));
1757       break;
1758
1759     case NE:
1760       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1761                                       const0_rtx));
1762       inverse = 1;
1763       break;
1764
1765     default:
1766       gcc_unreachable ();
1767     }
1768
1769   if (inverse)
1770     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1771                                     operands[1]));
1772   else
1773     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1774                                     operands[2]));
1775
1776   DONE;
1777 })
1778
1779 ;; Patterns for builtins.
1780
1781 ; good for plain vadd, vaddq.
1782
1783 (define_expand "neon_vadd<mode>"
1784   [(match_operand:VDQX 0 "s_register_operand" "=w")
1785    (match_operand:VDQX 1 "s_register_operand" "w")
1786    (match_operand:VDQX 2 "s_register_operand" "w")
1787    (match_operand:SI 3 "immediate_operand" "i")]
1788   "TARGET_NEON"
1789 {
1790   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1791     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1792   else
1793     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1794                                            operands[2]));
1795   DONE;
1796 })
1797
1798 ; Note that NEON operations don't support the full IEEE 754 standard: in
1799 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1800 ; use those instructions for autovectorization, etc. unless
1801 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1802 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1803 ; header) must work in either case: if -funsafe-math-optimizations is given,
1804 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1805 ; expand to unspecs (which may potentially limit the extent to which they might
1806 ; be optimized by generic code).
1807
1808 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1809
1810 (define_insn "neon_vadd<mode>_unspec"
1811   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1812         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1813                       (match_operand:VDQX 2 "s_register_operand" "w")]
1814                      UNSPEC_VADD))]
1815   "TARGET_NEON"
1816   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1817   [(set (attr "neon_type")
1818       (if_then_else (match_test "<Is_float_mode>")
1819                     (if_then_else (match_test "<Is_d_reg>")
1820                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1821                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1822                     (const_string "neon_int_1")))]
1823 )
1824
1825 ; operand 3 represents in bits:
1826 ;  bit 0: signed (vs unsigned).
1827 ;  bit 1: rounding (vs none).
1828
1829 (define_insn "neon_vaddl<mode>"
1830   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1831         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1832                            (match_operand:VDI 2 "s_register_operand" "w")
1833                            (match_operand:SI 3 "immediate_operand" "i")]
1834                           UNSPEC_VADDL))]
1835   "TARGET_NEON"
1836   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1837   [(set_attr "neon_type" "neon_int_3")]
1838 )
1839
1840 (define_insn "neon_vaddw<mode>"
1841   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1842         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1843                            (match_operand:VDI 2 "s_register_operand" "w")
1844                            (match_operand:SI 3 "immediate_operand" "i")]
1845                           UNSPEC_VADDW))]
1846   "TARGET_NEON"
1847   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1848   [(set_attr "neon_type" "neon_int_2")]
1849 )
1850
1851 ; vhadd and vrhadd.
1852
1853 (define_insn "neon_vhadd<mode>"
1854   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1855         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1856                        (match_operand:VDQIW 2 "s_register_operand" "w")
1857                        (match_operand:SI 3 "immediate_operand" "i")]
1858                       UNSPEC_VHADD))]
1859   "TARGET_NEON"
1860   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1861   [(set_attr "neon_type" "neon_int_4")]
1862 )
1863
1864 (define_insn "neon_vqadd<mode>"
1865   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1866         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1867                        (match_operand:VDQIX 2 "s_register_operand" "w")
1868                        (match_operand:SI 3 "immediate_operand" "i")]
1869                      UNSPEC_VQADD))]
1870   "TARGET_NEON"
1871   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1872   [(set_attr "neon_type" "neon_int_4")]
1873 )
1874
1875 (define_insn "neon_vaddhn<mode>"
1876   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1877         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1878                             (match_operand:VN 2 "s_register_operand" "w")
1879                             (match_operand:SI 3 "immediate_operand" "i")]
1880                            UNSPEC_VADDHN))]
1881   "TARGET_NEON"
1882   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1883   [(set_attr "neon_type" "neon_int_4")]
1884 )
1885
1886 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1887 ;; polynomial multiplication case that can specified by operand 3.
1888 (define_insn "neon_vmul<mode>"
1889   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1890         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1891                       (match_operand:VDQW 2 "s_register_operand" "w")
1892                       (match_operand:SI 3 "immediate_operand" "i")]
1893                      UNSPEC_VMUL))]
1894   "TARGET_NEON"
1895   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1896   [(set (attr "neon_type")
1897       (if_then_else (match_test "<Is_float_mode>")
1898                     (if_then_else (match_test "<Is_d_reg>")
1899                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1900                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1901                     (if_then_else (match_test "<Is_d_reg>")
1902                                   (if_then_else
1903                                     (match_test "<Scalar_mul_8_16>")
1904                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1905                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1906                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1907                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
1908                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1909 )
1910
1911 (define_expand "neon_vmla<mode>"
1912   [(match_operand:VDQW 0 "s_register_operand" "=w")
1913    (match_operand:VDQW 1 "s_register_operand" "0")
1914    (match_operand:VDQW 2 "s_register_operand" "w")
1915    (match_operand:VDQW 3 "s_register_operand" "w")
1916    (match_operand:SI 4 "immediate_operand" "i")]
1917   "TARGET_NEON"
1918 {
1919   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1920     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1921                                              operands[2], operands[3]));
1922   else
1923     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1924                                            operands[2], operands[3]));
1925   DONE;
1926 })
1927
1928 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1929
1930 (define_insn "neon_vmla<mode>_unspec"
1931   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1932         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1933                      (match_operand:VDQ 2 "s_register_operand" "w")
1934                      (match_operand:VDQ 3 "s_register_operand" "w")]
1935                     UNSPEC_VMLA))]
1936   "TARGET_NEON"
1937   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1938   [(set (attr "neon_type")
1939       (if_then_else (match_test "<Is_float_mode>")
1940                     (if_then_else (match_test "<Is_d_reg>")
1941                                   (const_string "neon_fp_vmla_ddd")
1942                                   (const_string "neon_fp_vmla_qqq"))
1943                     (if_then_else (match_test "<Is_d_reg>")
1944                                   (if_then_else
1945                                     (match_test "<Scalar_mul_8_16>")
1946                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1947                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1948                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1949                                     (const_string "neon_mla_qqq_8_16")
1950                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1951 )
1952
1953 (define_insn "neon_vmlal<mode>"
1954   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1955         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1956                            (match_operand:VW 2 "s_register_operand" "w")
1957                            (match_operand:VW 3 "s_register_operand" "w")
1958                            (match_operand:SI 4 "immediate_operand" "i")]
1959                           UNSPEC_VMLAL))]
1960   "TARGET_NEON"
1961   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1962   [(set (attr "neon_type")
1963      (if_then_else (match_test "<Scalar_mul_8_16>")
1964                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1965                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1966 )
1967
1968 (define_expand "neon_vmls<mode>"
1969   [(match_operand:VDQW 0 "s_register_operand" "=w")
1970    (match_operand:VDQW 1 "s_register_operand" "0")
1971    (match_operand:VDQW 2 "s_register_operand" "w")
1972    (match_operand:VDQW 3 "s_register_operand" "w")
1973    (match_operand:SI 4 "immediate_operand" "i")]
1974   "TARGET_NEON"
1975 {
1976   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1977     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1978                  operands[1], operands[2], operands[3]));
1979   else
1980     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1981                                            operands[2], operands[3]));
1982   DONE;
1983 })
1984
1985 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1986
1987 (define_insn "neon_vmls<mode>_unspec"
1988   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1989         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1990                      (match_operand:VDQ 2 "s_register_operand" "w")
1991                      (match_operand:VDQ 3 "s_register_operand" "w")]
1992                     UNSPEC_VMLS))]
1993   "TARGET_NEON"
1994   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1995   [(set (attr "neon_type")
1996       (if_then_else (match_test "<Is_float_mode>")
1997                     (if_then_else (match_test "<Is_d_reg>")
1998                                   (const_string "neon_fp_vmla_ddd")
1999                                   (const_string "neon_fp_vmla_qqq"))
2000                     (if_then_else (match_test "<Is_d_reg>")
2001                                   (if_then_else
2002                                     (match_test "<Scalar_mul_8_16>")
2003                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2004                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2005                                   (if_then_else
2006                                     (match_test "<Scalar_mul_8_16>")
2007                                     (const_string "neon_mla_qqq_8_16")
2008                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2009 )
2010
2011 (define_insn "neon_vmlsl<mode>"
2012   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2013         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2014                            (match_operand:VW 2 "s_register_operand" "w")
2015                            (match_operand:VW 3 "s_register_operand" "w")
2016                            (match_operand:SI 4 "immediate_operand" "i")]
2017                           UNSPEC_VMLSL))]
2018   "TARGET_NEON"
2019   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2020   [(set (attr "neon_type")
2021      (if_then_else (match_test "<Scalar_mul_8_16>")
2022                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2023                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2024 )
2025
2026 (define_insn "neon_vqdmulh<mode>"
2027   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2028         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2029                        (match_operand:VMDQI 2 "s_register_operand" "w")
2030                        (match_operand:SI 3 "immediate_operand" "i")]
2031                       UNSPEC_VQDMULH))]
2032   "TARGET_NEON"
2033   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2034   [(set (attr "neon_type")
2035      (if_then_else (match_test "<Is_d_reg>")
2036         (if_then_else (match_test "<Scalar_mul_8_16>")
2037                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2038                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2039         (if_then_else (match_test "<Scalar_mul_8_16>")
2040                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2041                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2042 )
2043
2044 (define_insn "neon_vqdmlal<mode>"
2045   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2046         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2047                            (match_operand:VMDI 2 "s_register_operand" "w")
2048                            (match_operand:VMDI 3 "s_register_operand" "w")
2049                            (match_operand:SI 4 "immediate_operand" "i")]
2050                           UNSPEC_VQDMLAL))]
2051   "TARGET_NEON"
2052   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2053   [(set (attr "neon_type")
2054      (if_then_else (match_test "<Scalar_mul_8_16>")
2055                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2056                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2057 )
2058
2059 (define_insn "neon_vqdmlsl<mode>"
2060   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2061         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2062                            (match_operand:VMDI 2 "s_register_operand" "w")
2063                            (match_operand:VMDI 3 "s_register_operand" "w")
2064                            (match_operand:SI 4 "immediate_operand" "i")]
2065                           UNSPEC_VQDMLSL))]
2066   "TARGET_NEON"
2067   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2068   [(set (attr "neon_type")
2069      (if_then_else (match_test "<Scalar_mul_8_16>")
2070                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2071                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2072 )
2073
2074 (define_insn "neon_vmull<mode>"
2075   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2076         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2077                            (match_operand:VW 2 "s_register_operand" "w")
2078                            (match_operand:SI 3 "immediate_operand" "i")]
2079                           UNSPEC_VMULL))]
2080   "TARGET_NEON"
2081   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2082   [(set (attr "neon_type")
2083      (if_then_else (match_test "<Scalar_mul_8_16>")
2084                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2085                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2086 )
2087
2088 (define_insn "neon_vqdmull<mode>"
2089   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2090         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2091                            (match_operand:VMDI 2 "s_register_operand" "w")
2092                            (match_operand:SI 3 "immediate_operand" "i")]
2093                           UNSPEC_VQDMULL))]
2094   "TARGET_NEON"
2095   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2096   [(set (attr "neon_type")
2097      (if_then_else (match_test "<Scalar_mul_8_16>")
2098                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2099                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2100 )
2101
2102 (define_expand "neon_vsub<mode>"
2103   [(match_operand:VDQX 0 "s_register_operand" "=w")
2104    (match_operand:VDQX 1 "s_register_operand" "w")
2105    (match_operand:VDQX 2 "s_register_operand" "w")
2106    (match_operand:SI 3 "immediate_operand" "i")]
2107   "TARGET_NEON"
2108 {
2109   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2110     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2111   else
2112     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2113                                            operands[2]));
2114   DONE;
2115 })
2116
2117 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2118
2119 (define_insn "neon_vsub<mode>_unspec"
2120   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2121         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2122                       (match_operand:VDQX 2 "s_register_operand" "w")]
2123                      UNSPEC_VSUB))]
2124   "TARGET_NEON"
2125   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2126   [(set (attr "neon_type")
2127       (if_then_else (match_test "<Is_float_mode>")
2128                     (if_then_else (match_test "<Is_d_reg>")
2129                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2130                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2131                     (const_string "neon_int_2")))]
2132 )
2133
2134 (define_insn "neon_vsubl<mode>"
2135   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2136         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2137                            (match_operand:VDI 2 "s_register_operand" "w")
2138                            (match_operand:SI 3 "immediate_operand" "i")]
2139                           UNSPEC_VSUBL))]
2140   "TARGET_NEON"
2141   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2142   [(set_attr "neon_type" "neon_int_2")]
2143 )
2144
2145 (define_insn "neon_vsubw<mode>"
2146   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2147         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2148                            (match_operand:VDI 2 "s_register_operand" "w")
2149                            (match_operand:SI 3 "immediate_operand" "i")]
2150                           UNSPEC_VSUBW))]
2151   "TARGET_NEON"
2152   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2153   [(set_attr "neon_type" "neon_int_2")]
2154 )
2155
2156 (define_insn "neon_vqsub<mode>"
2157   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2158         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2159                        (match_operand:VDQIX 2 "s_register_operand" "w")
2160                        (match_operand:SI 3 "immediate_operand" "i")]
2161                       UNSPEC_VQSUB))]
2162   "TARGET_NEON"
2163   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2164   [(set_attr "neon_type" "neon_int_5")]
2165 )
2166
2167 (define_insn "neon_vhsub<mode>"
2168   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2169         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2170                        (match_operand:VDQIW 2 "s_register_operand" "w")
2171                        (match_operand:SI 3 "immediate_operand" "i")]
2172                       UNSPEC_VHSUB))]
2173   "TARGET_NEON"
2174   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2175   [(set_attr "neon_type" "neon_int_5")]
2176 )
2177
2178 (define_insn "neon_vsubhn<mode>"
2179   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2180         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2181                             (match_operand:VN 2 "s_register_operand" "w")
2182                             (match_operand:SI 3 "immediate_operand" "i")]
2183                            UNSPEC_VSUBHN))]
2184   "TARGET_NEON"
2185   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2186   [(set_attr "neon_type" "neon_int_4")]
2187 )
2188
2189 (define_insn "neon_vceq<mode>"
2190   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2191         (unspec:<V_cmp_result>
2192           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2193            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2194            (match_operand:SI 3 "immediate_operand" "i,i")]
2195           UNSPEC_VCEQ))]
2196   "TARGET_NEON"
2197   "@
2198   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2199   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2200   [(set (attr "neon_type")
2201       (if_then_else (match_test "<Is_float_mode>")
2202                     (if_then_else (match_test "<Is_d_reg>")
2203                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2204                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2205                     (const_string "neon_int_5")))]
2206 )
2207
2208 (define_insn "neon_vcge<mode>"
2209   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2210         (unspec:<V_cmp_result>
2211           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2212            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2213            (match_operand:SI 3 "immediate_operand" "i,i")]
2214           UNSPEC_VCGE))]
2215   "TARGET_NEON"
2216   "@
2217   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2218   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2219   [(set (attr "neon_type")
2220      (if_then_else (match_test "<Is_float_mode>")
2221                    (if_then_else (match_test "<Is_d_reg>")
2222                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2223                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2224                    (const_string "neon_int_5")))]
2225 )
2226
2227 (define_insn "neon_vcgeu<mode>"
2228   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2229         (unspec:<V_cmp_result>
2230           [(match_operand:VDQIW 1 "s_register_operand" "w")
2231            (match_operand:VDQIW 2 "s_register_operand" "w")
2232            (match_operand:SI 3 "immediate_operand" "i")]
2233           UNSPEC_VCGEU))]
2234   "TARGET_NEON"
2235   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2236   [(set_attr "neon_type" "neon_int_5")]
2237 )
2238
2239 (define_insn "neon_vcgt<mode>"
2240   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2241         (unspec:<V_cmp_result>
2242           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2243            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2244            (match_operand:SI 3 "immediate_operand" "i,i")]
2245           UNSPEC_VCGT))]
2246   "TARGET_NEON"
2247   "@
2248   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2249   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2250   [(set (attr "neon_type")
2251      (if_then_else (match_test "<Is_float_mode>")
2252                    (if_then_else (match_test "<Is_d_reg>")
2253                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2254                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2255                    (const_string "neon_int_5")))]
2256 )
2257
2258 (define_insn "neon_vcgtu<mode>"
2259   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2260         (unspec:<V_cmp_result>
2261           [(match_operand:VDQIW 1 "s_register_operand" "w")
2262            (match_operand:VDQIW 2 "s_register_operand" "w")
2263            (match_operand:SI 3 "immediate_operand" "i")]
2264           UNSPEC_VCGTU))]
2265   "TARGET_NEON"
2266   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2267   [(set_attr "neon_type" "neon_int_5")]
2268 )
2269
2270 ;; VCLE and VCLT only support comparisons with immediate zero (register
2271 ;; variants are VCGE and VCGT with operands reversed).
2272
2273 (define_insn "neon_vcle<mode>"
2274   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2275         (unspec:<V_cmp_result>
2276           [(match_operand:VDQW 1 "s_register_operand" "w")
2277            (match_operand:VDQW 2 "zero_operand" "Dz")
2278            (match_operand:SI 3 "immediate_operand" "i")]
2279           UNSPEC_VCLE))]
2280   "TARGET_NEON"
2281   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2282   [(set (attr "neon_type")
2283       (if_then_else (match_test "<Is_float_mode>")
2284                     (if_then_else (match_test "<Is_d_reg>")
2285                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2286                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2287                     (const_string "neon_int_5")))]
2288 )
2289
2290 (define_insn "neon_vclt<mode>"
2291   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2292         (unspec:<V_cmp_result>
2293           [(match_operand:VDQW 1 "s_register_operand" "w")
2294            (match_operand:VDQW 2 "zero_operand" "Dz")
2295            (match_operand:SI 3 "immediate_operand" "i")]
2296           UNSPEC_VCLT))]
2297   "TARGET_NEON"
2298   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2299   [(set (attr "neon_type")
2300       (if_then_else (match_test "<Is_float_mode>")
2301                     (if_then_else (match_test "<Is_d_reg>")
2302                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2303                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2304                     (const_string "neon_int_5")))]
2305 )
2306
2307 (define_insn "neon_vcage<mode>"
2308   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2309         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2310                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2311                                 (match_operand:SI 3 "immediate_operand" "i")]
2312                                UNSPEC_VCAGE))]
2313   "TARGET_NEON"
2314   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2315   [(set (attr "neon_type")
2316      (if_then_else (match_test "<Is_d_reg>")
2317                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2318                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2319 )
2320
2321 (define_insn "neon_vcagt<mode>"
2322   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2323         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2324                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2325                                 (match_operand:SI 3 "immediate_operand" "i")]
2326                                UNSPEC_VCAGT))]
2327   "TARGET_NEON"
2328   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2329   [(set (attr "neon_type")
2330      (if_then_else (match_test "<Is_d_reg>")
2331                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2332                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2333 )
2334
2335 (define_insn "neon_vtst<mode>"
2336   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2337         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2338                        (match_operand:VDQIW 2 "s_register_operand" "w")
2339                        (match_operand:SI 3 "immediate_operand" "i")]
2340                       UNSPEC_VTST))]
2341   "TARGET_NEON"
2342   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2343   [(set_attr "neon_type" "neon_int_4")]
2344 )
2345
2346 (define_insn "neon_vabd<mode>"
2347   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2348         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2349                       (match_operand:VDQW 2 "s_register_operand" "w")
2350                       (match_operand:SI 3 "immediate_operand" "i")]
2351                      UNSPEC_VABD))]
2352   "TARGET_NEON"
2353   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2354   [(set (attr "neon_type")
2355      (if_then_else (match_test "<Is_float_mode>")
2356                    (if_then_else (match_test "<Is_d_reg>")
2357                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2358                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2359                    (const_string "neon_int_5")))]
2360 )
2361
2362 (define_insn "neon_vabdl<mode>"
2363   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2364         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2365                            (match_operand:VW 2 "s_register_operand" "w")
2366                            (match_operand:SI 3 "immediate_operand" "i")]
2367                           UNSPEC_VABDL))]
2368   "TARGET_NEON"
2369   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2370   [(set_attr "neon_type" "neon_int_5")]
2371 )
2372
2373 (define_insn "neon_vaba<mode>"
2374   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2375         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2376                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2377                                    (match_operand:SI 4 "immediate_operand" "i")]
2378                                   UNSPEC_VABD)
2379                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2380   "TARGET_NEON"
2381   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2382   [(set (attr "neon_type")
2383      (if_then_else (match_test "<Is_d_reg>")
2384                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2385 )
2386
2387 (define_insn "neon_vabal<mode>"
2388   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2389         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2390                                            (match_operand:VW 3 "s_register_operand" "w")
2391                                            (match_operand:SI 4 "immediate_operand" "i")]
2392                                            UNSPEC_VABDL)
2393                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2394   "TARGET_NEON"
2395   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2396   [(set_attr "neon_type" "neon_vaba")]
2397 )
2398
2399 (define_insn "neon_vmax<mode>"
2400   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2401         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2402                       (match_operand:VDQW 2 "s_register_operand" "w")
2403                       (match_operand:SI 3 "immediate_operand" "i")]
2404                      UNSPEC_VMAX))]
2405   "TARGET_NEON"
2406   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2407   [(set (attr "neon_type")
2408     (if_then_else (match_test "<Is_float_mode>")
2409                   (if_then_else (match_test "<Is_d_reg>")
2410                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2411                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2412                   (const_string "neon_int_5")))]
2413 )
2414
2415 (define_insn "neon_vmin<mode>"
2416   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2417         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2418                       (match_operand:VDQW 2 "s_register_operand" "w")
2419                       (match_operand:SI 3 "immediate_operand" "i")]
2420                      UNSPEC_VMIN))]
2421   "TARGET_NEON"
2422   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2423   [(set (attr "neon_type")
2424     (if_then_else (match_test "<Is_float_mode>")
2425                   (if_then_else (match_test "<Is_d_reg>")
2426                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2427                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2428                   (const_string "neon_int_5")))]
2429 )
2430
2431 (define_expand "neon_vpadd<mode>"
2432   [(match_operand:VD 0 "s_register_operand" "=w")
2433    (match_operand:VD 1 "s_register_operand" "w")
2434    (match_operand:VD 2 "s_register_operand" "w")
2435    (match_operand:SI 3 "immediate_operand" "i")]
2436   "TARGET_NEON"
2437 {
2438   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2439                                             operands[2]));
2440   DONE;
2441 })
2442
2443 (define_insn "neon_vpaddl<mode>"
2444   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2445         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2446                                   (match_operand:SI 2 "immediate_operand" "i")]
2447                                  UNSPEC_VPADDL))]
2448   "TARGET_NEON"
2449   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2450   ;; Assume this schedules like vaddl.
2451   [(set_attr "neon_type" "neon_int_3")]
2452 )
2453
2454 (define_insn "neon_vpadal<mode>"
2455   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2456         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2457                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2458                                   (match_operand:SI 3 "immediate_operand" "i")]
2459                                  UNSPEC_VPADAL))]
2460   "TARGET_NEON"
2461   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2462   ;; Assume this schedules like vpadd.
2463   [(set_attr "neon_type" "neon_int_1")]
2464 )
2465
2466 (define_insn "neon_vpmax<mode>"
2467   [(set (match_operand:VD 0 "s_register_operand" "=w")
2468         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2469                     (match_operand:VD 2 "s_register_operand" "w")
2470                     (match_operand:SI 3 "immediate_operand" "i")]
2471                    UNSPEC_VPMAX))]
2472   "TARGET_NEON"
2473   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2474   ;; Assume this schedules like vmax.
2475   [(set (attr "neon_type")
2476     (if_then_else (match_test "<Is_float_mode>")
2477                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2478                   (const_string "neon_int_5")))]
2479 )
2480
2481 (define_insn "neon_vpmin<mode>"
2482   [(set (match_operand:VD 0 "s_register_operand" "=w")
2483         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2484                     (match_operand:VD 2 "s_register_operand" "w")
2485                     (match_operand:SI 3 "immediate_operand" "i")]
2486                    UNSPEC_VPMIN))]
2487   "TARGET_NEON"
2488   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2489   ;; Assume this schedules like vmin.
2490   [(set (attr "neon_type")
2491     (if_then_else (match_test "<Is_float_mode>")
2492                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2493                   (const_string "neon_int_5")))]
2494 )
2495
2496 (define_insn "neon_vrecps<mode>"
2497   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2498         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2499                        (match_operand:VCVTF 2 "s_register_operand" "w")
2500                        (match_operand:SI 3 "immediate_operand" "i")]
2501                       UNSPEC_VRECPS))]
2502   "TARGET_NEON"
2503   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2504   [(set (attr "neon_type")
2505       (if_then_else (match_test "<Is_d_reg>")
2506                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2507                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2508 )
2509
2510 (define_insn "neon_vrsqrts<mode>"
2511   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2512         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2513                        (match_operand:VCVTF 2 "s_register_operand" "w")
2514                        (match_operand:SI 3 "immediate_operand" "i")]
2515                       UNSPEC_VRSQRTS))]
2516   "TARGET_NEON"
2517   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2518   [(set (attr "neon_type")
2519       (if_then_else (match_test "<Is_d_reg>")
2520                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2521                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2522 )
2523
2524 (define_expand "neon_vabs<mode>"
2525   [(match_operand:VDQW 0 "s_register_operand" "")
2526    (match_operand:VDQW 1 "s_register_operand" "")
2527    (match_operand:SI 2 "immediate_operand" "")]
2528   "TARGET_NEON"
2529 {
2530   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2531   DONE;
2532 })
2533
2534 (define_insn "neon_vqabs<mode>"
2535   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2536         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2537                        (match_operand:SI 2 "immediate_operand" "i")]
2538                       UNSPEC_VQABS))]
2539   "TARGET_NEON"
2540   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2541   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2542 )
2543
2544 (define_expand "neon_vneg<mode>"
2545   [(match_operand:VDQW 0 "s_register_operand" "")
2546    (match_operand:VDQW 1 "s_register_operand" "")
2547    (match_operand:SI 2 "immediate_operand" "")]
2548   "TARGET_NEON"
2549 {
2550   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2551   DONE;
2552 })
2553
2554 (define_insn "neon_vqneg<mode>"
2555   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2556         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2557                        (match_operand:SI 2 "immediate_operand" "i")]
2558                       UNSPEC_VQNEG))]
2559   "TARGET_NEON"
2560   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2561   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2562 )
2563
2564 (define_insn "neon_vcls<mode>"
2565   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2566         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2567                        (match_operand:SI 2 "immediate_operand" "i")]
2568                       UNSPEC_VCLS))]
2569   "TARGET_NEON"
2570   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2571   [(set_attr "neon_type" "neon_int_1")]
2572 )
2573
2574 (define_insn "clz<mode>2"
2575   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2576         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2577   "TARGET_NEON"
2578   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2579   [(set_attr "neon_type" "neon_int_1")]
2580 )
2581
2582 (define_expand "neon_vclz<mode>"
2583   [(match_operand:VDQIW 0 "s_register_operand" "")
2584    (match_operand:VDQIW 1 "s_register_operand" "")
2585    (match_operand:SI 2 "immediate_operand" "")]
2586   "TARGET_NEON"
2587 {
2588   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2589   DONE;
2590 })
2591
2592 (define_insn "popcount<mode>2"
2593   [(set (match_operand:VE 0 "s_register_operand" "=w")
2594         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2595   "TARGET_NEON"
2596   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2597   [(set_attr "neon_type" "neon_int_1")]
2598 )
2599
2600 (define_expand "neon_vcnt<mode>"
2601   [(match_operand:VE 0 "s_register_operand" "=w")
2602    (match_operand:VE 1 "s_register_operand" "w")
2603    (match_operand:SI 2 "immediate_operand" "i")]
2604   "TARGET_NEON"
2605 {
2606   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2607   DONE;
2608 })
2609
2610 (define_insn "neon_vrecpe<mode>"
2611   [(set (match_operand:V32 0 "s_register_operand" "=w")
2612         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2613                      (match_operand:SI 2 "immediate_operand" "i")]
2614                     UNSPEC_VRECPE))]
2615   "TARGET_NEON"
2616   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2617   [(set (attr "neon_type")
2618       (if_then_else (match_test "<Is_d_reg>")
2619                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2620                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2621 )
2622
2623 (define_insn "neon_vrsqrte<mode>"
2624   [(set (match_operand:V32 0 "s_register_operand" "=w")
2625         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2626                      (match_operand:SI 2 "immediate_operand" "i")]
2627                     UNSPEC_VRSQRTE))]
2628   "TARGET_NEON"
2629   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2630   [(set (attr "neon_type")
2631       (if_then_else (match_test "<Is_d_reg>")
2632                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2633                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2634 )
2635
2636 (define_expand "neon_vmvn<mode>"
2637   [(match_operand:VDQIW 0 "s_register_operand" "")
2638    (match_operand:VDQIW 1 "s_register_operand" "")
2639    (match_operand:SI 2 "immediate_operand" "")]
2640   "TARGET_NEON"
2641 {
2642   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2643   DONE;
2644 })
2645
2646 (define_insn "neon_vget_lane<mode>_sext_internal"
2647   [(set (match_operand:SI 0 "s_register_operand" "=r")
2648         (sign_extend:SI
2649           (vec_select:<V_elem>
2650             (match_operand:VD 1 "s_register_operand" "w")
2651             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2652   "TARGET_NEON"
2653 {
2654   if (BYTES_BIG_ENDIAN)
2655     {
2656       int elt = INTVAL (operands[2]);
2657       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2658       operands[2] = GEN_INT (elt);
2659     }
2660   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2661 }
2662   [(set_attr "neon_type" "neon_bp_simple")]
2663 )
2664
2665 (define_insn "neon_vget_lane<mode>_zext_internal"
2666   [(set (match_operand:SI 0 "s_register_operand" "=r")
2667         (zero_extend:SI
2668           (vec_select:<V_elem>
2669             (match_operand:VD 1 "s_register_operand" "w")
2670             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2671   "TARGET_NEON"
2672 {
2673   if (BYTES_BIG_ENDIAN)
2674     {
2675       int elt = INTVAL (operands[2]);
2676       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2677       operands[2] = GEN_INT (elt);
2678     }
2679   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2680 }
2681   [(set_attr "neon_type" "neon_bp_simple")]
2682 )
2683
2684 (define_insn "neon_vget_lane<mode>_sext_internal"
2685   [(set (match_operand:SI 0 "s_register_operand" "=r")
2686         (sign_extend:SI
2687           (vec_select:<V_elem>
2688             (match_operand:VQ 1 "s_register_operand" "w")
2689             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2690   "TARGET_NEON"
2691 {
2692   rtx ops[3];
2693   int regno = REGNO (operands[1]);
2694   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2695   unsigned int elt = INTVAL (operands[2]);
2696   unsigned int elt_adj = elt % halfelts;
2697
2698   if (BYTES_BIG_ENDIAN)
2699     elt_adj = halfelts - 1 - elt_adj;
2700
2701   ops[0] = operands[0];
2702   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2703   ops[2] = GEN_INT (elt_adj);
2704   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2705
2706   return "";
2707 }
2708   [(set_attr "neon_type" "neon_bp_simple")]
2709 )
2710
2711 (define_insn "neon_vget_lane<mode>_zext_internal"
2712   [(set (match_operand:SI 0 "s_register_operand" "=r")
2713         (zero_extend:SI
2714           (vec_select:<V_elem>
2715             (match_operand:VQ 1 "s_register_operand" "w")
2716             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2717   "TARGET_NEON"
2718 {
2719   rtx ops[3];
2720   int regno = REGNO (operands[1]);
2721   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2722   unsigned int elt = INTVAL (operands[2]);
2723   unsigned int elt_adj = elt % halfelts;
2724
2725   if (BYTES_BIG_ENDIAN)
2726     elt_adj = halfelts - 1 - elt_adj;
2727
2728   ops[0] = operands[0];
2729   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2730   ops[2] = GEN_INT (elt_adj);
2731   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2732
2733   return "";
2734 }
2735   [(set_attr "neon_type" "neon_bp_simple")]
2736 )
2737
2738 (define_expand "neon_vget_lane<mode>"
2739   [(match_operand:<V_ext> 0 "s_register_operand" "")
2740    (match_operand:VDQW 1 "s_register_operand" "")
2741    (match_operand:SI 2 "immediate_operand" "")
2742    (match_operand:SI 3 "immediate_operand" "")]
2743   "TARGET_NEON"
2744 {
2745   HOST_WIDE_INT magic = INTVAL (operands[3]);
2746   rtx insn;
2747
2748   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2749
2750   if (BYTES_BIG_ENDIAN)
2751     {
2752       /* The intrinsics are defined in terms of a model where the
2753          element ordering in memory is vldm order, whereas the generic
2754          RTL is defined in terms of a model where the element ordering
2755          in memory is array order.  Convert the lane number to conform
2756          to this model.  */
2757       unsigned int elt = INTVAL (operands[2]);
2758       unsigned int reg_nelts
2759         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2760       elt ^= reg_nelts - 1;
2761       operands[2] = GEN_INT (elt);
2762     }
2763
2764   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2765     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2766   else
2767     {
2768       if ((magic & 1) != 0)
2769         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2770                                                        operands[2]);
2771       else
2772         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2773                                                        operands[2]);
2774     }
2775   emit_insn (insn);
2776   DONE;
2777 })
2778
2779 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2780 ; elements.
2781
2782 (define_expand "neon_vget_lanedi"
2783   [(match_operand:DI 0 "s_register_operand" "=r")
2784    (match_operand:DI 1 "s_register_operand" "w")
2785    (match_operand:SI 2 "immediate_operand" "i")
2786    (match_operand:SI 3 "immediate_operand" "i")]
2787   "TARGET_NEON"
2788 {
2789   neon_lane_bounds (operands[2], 0, 1);
2790   emit_move_insn (operands[0], operands[1]);
2791   DONE;
2792 })
2793
2794 (define_expand "neon_vget_lanev2di"
2795   [(match_operand:DI 0 "s_register_operand" "")
2796    (match_operand:V2DI 1 "s_register_operand" "")
2797    (match_operand:SI 2 "immediate_operand" "")
2798    (match_operand:SI 3 "immediate_operand" "")]
2799   "TARGET_NEON"
2800 {
2801   switch (INTVAL (operands[2]))
2802     {
2803     case 0:
2804       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2805       break;
2806     case 1:
2807       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2808       break;
2809     default:
2810       neon_lane_bounds (operands[2], 0, 1);
2811       FAIL;
2812     }
2813   DONE;
2814 })
2815
2816 (define_expand "neon_vset_lane<mode>"
2817   [(match_operand:VDQ 0 "s_register_operand" "=w")
2818    (match_operand:<V_elem> 1 "s_register_operand" "r")
2819    (match_operand:VDQ 2 "s_register_operand" "0")
2820    (match_operand:SI 3 "immediate_operand" "i")]
2821   "TARGET_NEON"
2822 {
2823   unsigned int elt = INTVAL (operands[3]);
2824   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2825
2826   if (BYTES_BIG_ENDIAN)
2827     {
2828       unsigned int reg_nelts
2829         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2830       elt ^= reg_nelts - 1;
2831     }
2832
2833   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2834                                          GEN_INT (1 << elt), operands[2]));
2835   DONE;
2836 })
2837
2838 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2839
2840 (define_expand "neon_vset_lanedi"
2841   [(match_operand:DI 0 "s_register_operand" "=w")
2842    (match_operand:DI 1 "s_register_operand" "r")
2843    (match_operand:DI 2 "s_register_operand" "0")
2844    (match_operand:SI 3 "immediate_operand" "i")]
2845   "TARGET_NEON"
2846 {
2847   neon_lane_bounds (operands[3], 0, 1);
2848   emit_move_insn (operands[0], operands[1]);
2849   DONE;
2850 })
2851
2852 (define_expand "neon_vcreate<mode>"
2853   [(match_operand:VDX 0 "s_register_operand" "")
2854    (match_operand:DI 1 "general_operand" "")]
2855   "TARGET_NEON"
2856 {
2857   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2858   emit_move_insn (operands[0], src);
2859   DONE;
2860 })
2861
2862 (define_insn "neon_vdup_n<mode>"
2863   [(set (match_operand:VX 0 "s_register_operand" "=w")
2864         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2865   "TARGET_NEON"
2866   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2867   ;; Assume this schedules like vmov.
2868   [(set_attr "neon_type" "neon_bp_simple")]
2869 )
2870
2871 (define_insn "neon_vdup_n<mode>"
2872   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2873         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2874   "TARGET_NEON"
2875   "@
2876   vdup.<V_sz_elem>\t%<V_reg>0, %1
2877   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
2878   ;; Assume this schedules like vmov.
2879   [(set_attr "neon_type" "neon_bp_simple")]
2880 )
2881
2882 (define_expand "neon_vdup_ndi"
2883   [(match_operand:DI 0 "s_register_operand" "=w")
2884    (match_operand:DI 1 "s_register_operand" "r")]
2885   "TARGET_NEON"
2886 {
2887   emit_move_insn (operands[0], operands[1]);
2888   DONE;
2889 }
2890 )
2891
2892 (define_insn "neon_vdup_nv2di"
2893   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2894         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2895   "TARGET_NEON"
2896   "@
2897   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
2898   vmov\t%e0, %P1\;vmov\t%f0, %P1"
2899   [(set_attr "length" "8")
2900    (set_attr "neon_type" "neon_bp_simple")]
2901 )
2902
2903 (define_insn "neon_vdup_lane<mode>_internal"
2904   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2905         (vec_duplicate:VDQW
2906           (vec_select:<V_elem>
2907             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2908             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2909   "TARGET_NEON"
2910 {
2911   if (BYTES_BIG_ENDIAN)
2912     {
2913       int elt = INTVAL (operands[2]);
2914       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2915       operands[2] = GEN_INT (elt);
2916     }
2917   if (<Is_d_reg>)
2918     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2919   else
2920     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2921 }
2922   ;; Assume this schedules like vmov.
2923   [(set_attr "neon_type" "neon_bp_simple")]
2924 )
2925
2926 (define_expand "neon_vdup_lane<mode>"
2927   [(match_operand:VDQW 0 "s_register_operand" "=w")
2928    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2929    (match_operand:SI 2 "immediate_operand" "i")]
2930   "TARGET_NEON"
2931 {
2932   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2933   if (BYTES_BIG_ENDIAN)
2934     {
2935       unsigned int elt = INTVAL (operands[2]);
2936       unsigned int reg_nelts
2937         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2938       elt ^= reg_nelts - 1;
2939       operands[2] = GEN_INT (elt);
2940     }
2941     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2942                                                   operands[2]));
2943     DONE;
2944 })
2945
2946 ; Scalar index is ignored, since only zero is valid here.
2947 (define_expand "neon_vdup_lanedi"
2948   [(match_operand:DI 0 "s_register_operand" "=w")
2949    (match_operand:DI 1 "s_register_operand" "w")
2950    (match_operand:SI 2 "immediate_operand" "i")]
2951   "TARGET_NEON"
2952 {
2953   neon_lane_bounds (operands[2], 0, 1);
2954   emit_move_insn (operands[0], operands[1]);
2955   DONE;
2956 })
2957
2958 ; Likewise for v2di, as the DImode second operand has only a single element.
2959 (define_expand "neon_vdup_lanev2di"
2960   [(match_operand:V2DI 0 "s_register_operand" "=w")
2961    (match_operand:DI 1 "s_register_operand" "w")
2962    (match_operand:SI 2 "immediate_operand" "i")]
2963   "TARGET_NEON"
2964 {
2965   neon_lane_bounds (operands[2], 0, 1);
2966   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2967   DONE;
2968 })
2969
2970 ; Disabled before reload because we don't want combine doing something silly,
2971 ; but used by the post-reload expansion of neon_vcombine.
2972 (define_insn "*neon_vswp<mode>"
2973   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2974         (match_operand:VDQX 1 "s_register_operand" "+w"))
2975    (set (match_dup 1) (match_dup 0))]
2976   "TARGET_NEON && reload_completed"
2977   "vswp\t%<V_reg>0, %<V_reg>1"
2978   [(set (attr "neon_type")
2979         (if_then_else (match_test "<Is_d_reg>")
2980                       (const_string "neon_bp_simple")
2981                       (const_string "neon_bp_2cycle")))]
2982 )
2983
2984 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2985 ;; dest vector.
2986 ;; FIXME: A different implementation of this builtin could make it much
2987 ;; more likely that we wouldn't actually need to output anything (we could make
2988 ;; it so that the reg allocator puts things in the right places magically
2989 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2990
2991 (define_insn_and_split "neon_vcombine<mode>"
2992   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2993         (vec_concat:<V_DOUBLE>
2994           (match_operand:VDX 1 "s_register_operand" "w")
2995           (match_operand:VDX 2 "s_register_operand" "w")))]
2996   "TARGET_NEON"
2997   "#"
2998   "&& reload_completed"
2999   [(const_int 0)]
3000 {
3001   neon_split_vcombine (operands);
3002   DONE;
3003 })
3004
3005 (define_expand "neon_vget_high<mode>"
3006   [(match_operand:<V_HALF> 0 "s_register_operand")
3007    (match_operand:VQX 1 "s_register_operand")]
3008   "TARGET_NEON"
3009 {
3010   emit_move_insn (operands[0],
3011                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
3012                                        GET_MODE_SIZE (<V_HALF>mode)));
3013   DONE;
3014 })
3015
3016 (define_expand "neon_vget_low<mode>"
3017   [(match_operand:<V_HALF> 0 "s_register_operand")
3018    (match_operand:VQX 1 "s_register_operand")]
3019   "TARGET_NEON"
3020 {
3021   emit_move_insn (operands[0],
3022                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3023                                        <MODE>mode, 0));
3024   DONE;
3025 })
3026
3027 (define_insn "float<mode><V_cvtto>2"
3028   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3029         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3030   "TARGET_NEON && !flag_rounding_math"
3031   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3032   [(set (attr "neon_type")
3033      (if_then_else (match_test "<Is_d_reg>")
3034                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3035                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3036 )
3037
3038 (define_insn "floatuns<mode><V_cvtto>2"
3039   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3040         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3041   "TARGET_NEON && !flag_rounding_math"
3042   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3043   [(set (attr "neon_type")
3044      (if_then_else (match_test "<Is_d_reg>")
3045                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3046                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3047 )
3048
3049 (define_insn "fix_trunc<mode><V_cvtto>2"
3050   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3051         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3052   "TARGET_NEON"
3053   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3054   [(set (attr "neon_type")
3055      (if_then_else (match_test "<Is_d_reg>")
3056                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3057                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3058 )
3059
3060 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3061   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3062         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3063   "TARGET_NEON"
3064   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3065   [(set (attr "neon_type")
3066      (if_then_else (match_test "<Is_d_reg>")
3067                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3068                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3069 )
3070
3071 (define_insn "neon_vcvt<mode>"
3072   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3073         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3074                            (match_operand:SI 2 "immediate_operand" "i")]
3075                           UNSPEC_VCVT))]
3076   "TARGET_NEON"
3077   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3078   [(set (attr "neon_type")
3079      (if_then_else (match_test "<Is_d_reg>")
3080                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3081                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3082 )
3083
3084 (define_insn "neon_vcvt<mode>"
3085   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3086         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3087                            (match_operand:SI 2 "immediate_operand" "i")]
3088                           UNSPEC_VCVT))]
3089   "TARGET_NEON"
3090   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3091   [(set (attr "neon_type")
3092      (if_then_else (match_test "<Is_d_reg>")
3093                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3094                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3095 )
3096
3097 (define_insn "neon_vcvt_n<mode>"
3098   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3099         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3100                            (match_operand:SI 2 "immediate_operand" "i")
3101                            (match_operand:SI 3 "immediate_operand" "i")]
3102                           UNSPEC_VCVT_N))]
3103   "TARGET_NEON"
3104 {
3105   neon_const_bounds (operands[2], 1, 33);
3106   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3107 }
3108   [(set (attr "neon_type")
3109      (if_then_else (match_test "<Is_d_reg>")
3110                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3111                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3112 )
3113
3114 (define_insn "neon_vcvt_n<mode>"
3115   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3116         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3117                            (match_operand:SI 2 "immediate_operand" "i")
3118                            (match_operand:SI 3 "immediate_operand" "i")]
3119                           UNSPEC_VCVT_N))]
3120   "TARGET_NEON"
3121 {
3122   neon_const_bounds (operands[2], 1, 33);
3123   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3124 }
3125   [(set (attr "neon_type")
3126      (if_then_else (match_test "<Is_d_reg>")
3127                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3128                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3129 )
3130
3131 (define_insn "neon_vmovn<mode>"
3132   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3133         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3134                             (match_operand:SI 2 "immediate_operand" "i")]
3135                            UNSPEC_VMOVN))]
3136   "TARGET_NEON"
3137   "vmovn.<V_if_elem>\t%P0, %q1"
3138   [(set_attr "neon_type" "neon_bp_simple")]
3139 )
3140
3141 (define_insn "neon_vqmovn<mode>"
3142   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3143         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3144                             (match_operand:SI 2 "immediate_operand" "i")]
3145                            UNSPEC_VQMOVN))]
3146   "TARGET_NEON"
3147   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3148   [(set_attr "neon_type" "neon_shift_2")]
3149 )
3150
3151 (define_insn "neon_vqmovun<mode>"
3152   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3153         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3154                             (match_operand:SI 2 "immediate_operand" "i")]
3155                            UNSPEC_VQMOVUN))]
3156   "TARGET_NEON"
3157   "vqmovun.<V_s_elem>\t%P0, %q1"
3158   [(set_attr "neon_type" "neon_shift_2")]
3159 )
3160
3161 (define_insn "neon_vmovl<mode>"
3162   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3163         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3164                            (match_operand:SI 2 "immediate_operand" "i")]
3165                           UNSPEC_VMOVL))]
3166   "TARGET_NEON"
3167   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3168   [(set_attr "neon_type" "neon_shift_1")]
3169 )
3170
3171 (define_insn "neon_vmul_lane<mode>"
3172   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3173         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3174                      (match_operand:VMD 2 "s_register_operand"
3175                                         "<scalar_mul_constraint>")
3176                      (match_operand:SI 3 "immediate_operand" "i")
3177                      (match_operand:SI 4 "immediate_operand" "i")]
3178                     UNSPEC_VMUL_LANE))]
3179   "TARGET_NEON"
3180 {
3181   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3182   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3183 }
3184   [(set (attr "neon_type")
3185      (if_then_else (match_test "<Is_float_mode>")
3186                    (const_string "neon_fp_vmul_ddd")
3187                    (if_then_else (match_test "<Scalar_mul_8_16>")
3188                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3189                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3190 )
3191
3192 (define_insn "neon_vmul_lane<mode>"
3193   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3194         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3195                      (match_operand:<V_HALF> 2 "s_register_operand"
3196                                              "<scalar_mul_constraint>")
3197                      (match_operand:SI 3 "immediate_operand" "i")
3198                      (match_operand:SI 4 "immediate_operand" "i")]
3199                     UNSPEC_VMUL_LANE))]
3200   "TARGET_NEON"
3201 {
3202   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3203   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3204 }
3205   [(set (attr "neon_type")
3206      (if_then_else (match_test "<Is_float_mode>")
3207                    (const_string "neon_fp_vmul_qqd")
3208                    (if_then_else (match_test "<Scalar_mul_8_16>")
3209                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3210                                  (const_string "neon_mul_qqd_32_scalar"))))]
3211 )
3212
3213 (define_insn "neon_vmull_lane<mode>"
3214   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3215         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3216                            (match_operand:VMDI 2 "s_register_operand"
3217                                                "<scalar_mul_constraint>")
3218                            (match_operand:SI 3 "immediate_operand" "i")
3219                            (match_operand:SI 4 "immediate_operand" "i")]
3220                           UNSPEC_VMULL_LANE))]
3221   "TARGET_NEON"
3222 {
3223   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3224   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3225 }
3226   [(set (attr "neon_type")
3227      (if_then_else (match_test "<Scalar_mul_8_16>")
3228                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3229                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3230 )
3231
3232 (define_insn "neon_vqdmull_lane<mode>"
3233   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3234         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3235                            (match_operand:VMDI 2 "s_register_operand"
3236                                                "<scalar_mul_constraint>")
3237                            (match_operand:SI 3 "immediate_operand" "i")
3238                            (match_operand:SI 4 "immediate_operand" "i")]
3239                           UNSPEC_VQDMULL_LANE))]
3240   "TARGET_NEON"
3241 {
3242   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3243   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3244 }
3245   [(set (attr "neon_type")
3246      (if_then_else (match_test "<Scalar_mul_8_16>")
3247                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3248                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3249 )
3250
3251 (define_insn "neon_vqdmulh_lane<mode>"
3252   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3253         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3254                       (match_operand:<V_HALF> 2 "s_register_operand"
3255                                               "<scalar_mul_constraint>")
3256                       (match_operand:SI 3 "immediate_operand" "i")
3257                       (match_operand:SI 4 "immediate_operand" "i")]
3258                       UNSPEC_VQDMULH_LANE))]
3259   "TARGET_NEON"
3260 {
3261   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3262   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3263 }
3264   [(set (attr "neon_type")
3265      (if_then_else (match_test "<Scalar_mul_8_16>")
3266                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3267                    (const_string "neon_mul_qqd_32_scalar")))]
3268 )
3269
3270 (define_insn "neon_vqdmulh_lane<mode>"
3271   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3272         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3273                       (match_operand:VMDI 2 "s_register_operand"
3274                                           "<scalar_mul_constraint>")
3275                       (match_operand:SI 3 "immediate_operand" "i")
3276                       (match_operand:SI 4 "immediate_operand" "i")]
3277                       UNSPEC_VQDMULH_LANE))]
3278   "TARGET_NEON"
3279 {
3280   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3281   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3282 }
3283   [(set (attr "neon_type")
3284      (if_then_else (match_test "<Scalar_mul_8_16>")
3285                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3286                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3287 )
3288
3289 (define_insn "neon_vmla_lane<mode>"
3290   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3291         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3292                      (match_operand:VMD 2 "s_register_operand" "w")
3293                      (match_operand:VMD 3 "s_register_operand"
3294                                         "<scalar_mul_constraint>")
3295                      (match_operand:SI 4 "immediate_operand" "i")
3296                      (match_operand:SI 5 "immediate_operand" "i")]
3297                      UNSPEC_VMLA_LANE))]
3298   "TARGET_NEON"
3299 {
3300   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3301   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3302 }
3303   [(set (attr "neon_type")
3304      (if_then_else (match_test "<Is_float_mode>")
3305                    (const_string "neon_fp_vmla_ddd_scalar")
3306                    (if_then_else (match_test "<Scalar_mul_8_16>")
3307                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3308                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3309 )
3310
3311 (define_insn "neon_vmla_lane<mode>"
3312   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3313         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3314                      (match_operand:VMQ 2 "s_register_operand" "w")
3315                      (match_operand:<V_HALF> 3 "s_register_operand"
3316                                              "<scalar_mul_constraint>")
3317                      (match_operand:SI 4 "immediate_operand" "i")
3318                      (match_operand:SI 5 "immediate_operand" "i")]
3319                      UNSPEC_VMLA_LANE))]
3320   "TARGET_NEON"
3321 {
3322   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3323   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3324 }
3325   [(set (attr "neon_type")
3326      (if_then_else (match_test "<Is_float_mode>")
3327                    (const_string "neon_fp_vmla_qqq_scalar")
3328                    (if_then_else (match_test "<Scalar_mul_8_16>")
3329                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3330                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3331 )
3332
3333 (define_insn "neon_vmlal_lane<mode>"
3334   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3335         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3336                            (match_operand:VMDI 2 "s_register_operand" "w")
3337                            (match_operand:VMDI 3 "s_register_operand"
3338                                                "<scalar_mul_constraint>")
3339                            (match_operand:SI 4 "immediate_operand" "i")
3340                            (match_operand:SI 5 "immediate_operand" "i")]
3341                           UNSPEC_VMLAL_LANE))]
3342   "TARGET_NEON"
3343 {
3344   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3345   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3346 }
3347   [(set (attr "neon_type")
3348      (if_then_else (match_test "<Scalar_mul_8_16>")
3349                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3350                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3351 )
3352
3353 (define_insn "neon_vqdmlal_lane<mode>"
3354   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3355         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3356                            (match_operand:VMDI 2 "s_register_operand" "w")
3357                            (match_operand:VMDI 3 "s_register_operand"
3358                                                "<scalar_mul_constraint>")
3359                            (match_operand:SI 4 "immediate_operand" "i")
3360                            (match_operand:SI 5 "immediate_operand" "i")]
3361                           UNSPEC_VQDMLAL_LANE))]
3362   "TARGET_NEON"
3363 {
3364   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3365   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3366 }
3367   [(set (attr "neon_type")
3368      (if_then_else (match_test "<Scalar_mul_8_16>")
3369                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3370                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3371 )
3372
3373 (define_insn "neon_vmls_lane<mode>"
3374   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3375         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3376                      (match_operand:VMD 2 "s_register_operand" "w")
3377                      (match_operand:VMD 3 "s_register_operand"
3378                                         "<scalar_mul_constraint>")
3379                      (match_operand:SI 4 "immediate_operand" "i")
3380                      (match_operand:SI 5 "immediate_operand" "i")]
3381                     UNSPEC_VMLS_LANE))]
3382   "TARGET_NEON"
3383 {
3384   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3385   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3386 }
3387   [(set (attr "neon_type")
3388      (if_then_else (match_test "<Is_float_mode>")
3389                    (const_string "neon_fp_vmla_ddd_scalar")
3390                    (if_then_else (match_test "<Scalar_mul_8_16>")
3391                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3392                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3393 )
3394
3395 (define_insn "neon_vmls_lane<mode>"
3396   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3397         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3398                      (match_operand:VMQ 2 "s_register_operand" "w")
3399                      (match_operand:<V_HALF> 3 "s_register_operand"
3400                                              "<scalar_mul_constraint>")
3401                      (match_operand:SI 4 "immediate_operand" "i")
3402                      (match_operand:SI 5 "immediate_operand" "i")]
3403                     UNSPEC_VMLS_LANE))]
3404   "TARGET_NEON"
3405 {
3406   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3407   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3408 }
3409   [(set (attr "neon_type")
3410      (if_then_else (match_test "<Is_float_mode>")
3411                    (const_string "neon_fp_vmla_qqq_scalar")
3412                    (if_then_else (match_test "<Scalar_mul_8_16>")
3413                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3414                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3415 )
3416
3417 (define_insn "neon_vmlsl_lane<mode>"
3418   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3419         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3420                            (match_operand:VMDI 2 "s_register_operand" "w")
3421                            (match_operand:VMDI 3 "s_register_operand"
3422                                                "<scalar_mul_constraint>")
3423                            (match_operand:SI 4 "immediate_operand" "i")
3424                            (match_operand:SI 5 "immediate_operand" "i")]
3425                           UNSPEC_VMLSL_LANE))]
3426   "TARGET_NEON"
3427 {
3428   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3429   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3430 }
3431   [(set (attr "neon_type")
3432      (if_then_else (match_test "<Scalar_mul_8_16>")
3433                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3434                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3435 )
3436
3437 (define_insn "neon_vqdmlsl_lane<mode>"
3438   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3439         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3440                            (match_operand:VMDI 2 "s_register_operand" "w")
3441                            (match_operand:VMDI 3 "s_register_operand"
3442                                                "<scalar_mul_constraint>")
3443                            (match_operand:SI 4 "immediate_operand" "i")
3444                            (match_operand:SI 5 "immediate_operand" "i")]
3445                           UNSPEC_VQDMLSL_LANE))]
3446   "TARGET_NEON"
3447 {
3448   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3449   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3450 }
3451   [(set (attr "neon_type")
3452      (if_then_else (match_test "<Scalar_mul_8_16>")
3453                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3454                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3455 )
3456
3457 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3458 ; core register into a temp register, then use a scalar taken from that. This
3459 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3460 ; or extracted from another vector. The latter case it's currently better to
3461 ; use the "_lane" variant, and the former case can probably be implemented
3462 ; using vld1_lane, but that hasn't been done yet.
3463
3464 (define_expand "neon_vmul_n<mode>"
3465   [(match_operand:VMD 0 "s_register_operand" "")
3466    (match_operand:VMD 1 "s_register_operand" "")
3467    (match_operand:<V_elem> 2 "s_register_operand" "")
3468    (match_operand:SI 3 "immediate_operand" "")]
3469   "TARGET_NEON"
3470 {
3471   rtx tmp = gen_reg_rtx (<MODE>mode);
3472   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3473   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3474                                        const0_rtx, const0_rtx));
3475   DONE;
3476 })
3477
3478 (define_expand "neon_vmul_n<mode>"
3479   [(match_operand:VMQ 0 "s_register_operand" "")
3480    (match_operand:VMQ 1 "s_register_operand" "")
3481    (match_operand:<V_elem> 2 "s_register_operand" "")
3482    (match_operand:SI 3 "immediate_operand" "")]
3483   "TARGET_NEON"
3484 {
3485   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3486   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3487   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3488                                        const0_rtx, const0_rtx));
3489   DONE;
3490 })
3491
3492 (define_expand "neon_vmull_n<mode>"
3493   [(match_operand:<V_widen> 0 "s_register_operand" "")
3494    (match_operand:VMDI 1 "s_register_operand" "")
3495    (match_operand:<V_elem> 2 "s_register_operand" "")
3496    (match_operand:SI 3 "immediate_operand" "")]
3497   "TARGET_NEON"
3498 {
3499   rtx tmp = gen_reg_rtx (<MODE>mode);
3500   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3501   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3502                                         const0_rtx, operands[3]));
3503   DONE;
3504 })
3505
3506 (define_expand "neon_vqdmull_n<mode>"
3507   [(match_operand:<V_widen> 0 "s_register_operand" "")
3508    (match_operand:VMDI 1 "s_register_operand" "")
3509    (match_operand:<V_elem> 2 "s_register_operand" "")
3510    (match_operand:SI 3 "immediate_operand" "")]
3511   "TARGET_NEON"
3512 {
3513   rtx tmp = gen_reg_rtx (<MODE>mode);
3514   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3515   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3516                                           const0_rtx, const0_rtx));
3517   DONE;
3518 })
3519
3520 (define_expand "neon_vqdmulh_n<mode>"
3521   [(match_operand:VMDI 0 "s_register_operand" "")
3522    (match_operand:VMDI 1 "s_register_operand" "")
3523    (match_operand:<V_elem> 2 "s_register_operand" "")
3524    (match_operand:SI 3 "immediate_operand" "")]
3525   "TARGET_NEON"
3526 {
3527   rtx tmp = gen_reg_rtx (<MODE>mode);
3528   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3529   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3530                                           const0_rtx, operands[3]));
3531   DONE;
3532 })
3533
3534 (define_expand "neon_vqdmulh_n<mode>"
3535   [(match_operand:VMQI 0 "s_register_operand" "")
3536    (match_operand:VMQI 1 "s_register_operand" "")
3537    (match_operand:<V_elem> 2 "s_register_operand" "")
3538    (match_operand:SI 3 "immediate_operand" "")]
3539   "TARGET_NEON"
3540 {
3541   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3542   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3543   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3544                                           const0_rtx, operands[3]));
3545   DONE;
3546 })
3547
3548 (define_expand "neon_vmla_n<mode>"
3549   [(match_operand:VMD 0 "s_register_operand" "")
3550    (match_operand:VMD 1 "s_register_operand" "")
3551    (match_operand:VMD 2 "s_register_operand" "")
3552    (match_operand:<V_elem> 3 "s_register_operand" "")
3553    (match_operand:SI 4 "immediate_operand" "")]
3554   "TARGET_NEON"
3555 {
3556   rtx tmp = gen_reg_rtx (<MODE>mode);
3557   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3558   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3559                                        tmp, const0_rtx, operands[4]));
3560   DONE;
3561 })
3562
3563 (define_expand "neon_vmla_n<mode>"
3564   [(match_operand:VMQ 0 "s_register_operand" "")
3565    (match_operand:VMQ 1 "s_register_operand" "")
3566    (match_operand:VMQ 2 "s_register_operand" "")
3567    (match_operand:<V_elem> 3 "s_register_operand" "")
3568    (match_operand:SI 4 "immediate_operand" "")]
3569   "TARGET_NEON"
3570 {
3571   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3572   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3573   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3574                                        tmp, const0_rtx, operands[4]));
3575   DONE;
3576 })
3577
3578 (define_expand "neon_vmlal_n<mode>"
3579   [(match_operand:<V_widen> 0 "s_register_operand" "")
3580    (match_operand:<V_widen> 1 "s_register_operand" "")
3581    (match_operand:VMDI 2 "s_register_operand" "")
3582    (match_operand:<V_elem> 3 "s_register_operand" "")
3583    (match_operand:SI 4 "immediate_operand" "")]
3584   "TARGET_NEON"
3585 {
3586   rtx tmp = gen_reg_rtx (<MODE>mode);
3587   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3588   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3589                                         tmp, const0_rtx, operands[4]));
3590   DONE;
3591 })
3592
3593 (define_expand "neon_vqdmlal_n<mode>"
3594   [(match_operand:<V_widen> 0 "s_register_operand" "")
3595    (match_operand:<V_widen> 1 "s_register_operand" "")
3596    (match_operand:VMDI 2 "s_register_operand" "")
3597    (match_operand:<V_elem> 3 "s_register_operand" "")
3598    (match_operand:SI 4 "immediate_operand" "")]
3599   "TARGET_NEON"
3600 {
3601   rtx tmp = gen_reg_rtx (<MODE>mode);
3602   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3603   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3604                                           tmp, const0_rtx, operands[4]));
3605   DONE;
3606 })
3607
3608 (define_expand "neon_vmls_n<mode>"
3609   [(match_operand:VMD 0 "s_register_operand" "")
3610    (match_operand:VMD 1 "s_register_operand" "")
3611    (match_operand:VMD 2 "s_register_operand" "")
3612    (match_operand:<V_elem> 3 "s_register_operand" "")
3613    (match_operand:SI 4 "immediate_operand" "")]
3614   "TARGET_NEON"
3615 {
3616   rtx tmp = gen_reg_rtx (<MODE>mode);
3617   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3618   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3619                                        tmp, const0_rtx, operands[4]));
3620   DONE;
3621 })
3622
3623 (define_expand "neon_vmls_n<mode>"
3624   [(match_operand:VMQ 0 "s_register_operand" "")
3625    (match_operand:VMQ 1 "s_register_operand" "")
3626    (match_operand:VMQ 2 "s_register_operand" "")
3627    (match_operand:<V_elem> 3 "s_register_operand" "")
3628    (match_operand:SI 4 "immediate_operand" "")]
3629   "TARGET_NEON"
3630 {
3631   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3632   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3633   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3634                                        tmp, const0_rtx, operands[4]));
3635   DONE;
3636 })
3637
3638 (define_expand "neon_vmlsl_n<mode>"
3639   [(match_operand:<V_widen> 0 "s_register_operand" "")
3640    (match_operand:<V_widen> 1 "s_register_operand" "")
3641    (match_operand:VMDI 2 "s_register_operand" "")
3642    (match_operand:<V_elem> 3 "s_register_operand" "")
3643    (match_operand:SI 4 "immediate_operand" "")]
3644   "TARGET_NEON"
3645 {
3646   rtx tmp = gen_reg_rtx (<MODE>mode);
3647   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3648   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3649                                         tmp, const0_rtx, operands[4]));
3650   DONE;
3651 })
3652
3653 (define_expand "neon_vqdmlsl_n<mode>"
3654   [(match_operand:<V_widen> 0 "s_register_operand" "")
3655    (match_operand:<V_widen> 1 "s_register_operand" "")
3656    (match_operand:VMDI 2 "s_register_operand" "")
3657    (match_operand:<V_elem> 3 "s_register_operand" "")
3658    (match_operand:SI 4 "immediate_operand" "")]
3659   "TARGET_NEON"
3660 {
3661   rtx tmp = gen_reg_rtx (<MODE>mode);
3662   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3663   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3664                                           tmp, const0_rtx, operands[4]));
3665   DONE;
3666 })
3667
3668 (define_insn "neon_vext<mode>"
3669   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3670         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3671                       (match_operand:VDQX 2 "s_register_operand" "w")
3672                       (match_operand:SI 3 "immediate_operand" "i")]
3673                      UNSPEC_VEXT))]
3674   "TARGET_NEON"
3675 {
3676   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3677   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3678 }
3679   [(set (attr "neon_type")
3680       (if_then_else (match_test "<Is_d_reg>")
3681                     (const_string "neon_bp_simple")
3682                     (const_string "neon_bp_2cycle")))]
3683 )
3684
3685 (define_insn "neon_vrev64<mode>"
3686   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3687         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3688                      (match_operand:SI 2 "immediate_operand" "i")]
3689                     UNSPEC_VREV64))]
3690   "TARGET_NEON"
3691   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3692   [(set_attr "neon_type" "neon_bp_simple")]
3693 )
3694
3695 (define_insn "neon_vrev32<mode>"
3696   [(set (match_operand:VX 0 "s_register_operand" "=w")
3697         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3698                     (match_operand:SI 2 "immediate_operand" "i")]
3699                    UNSPEC_VREV32))]
3700   "TARGET_NEON"
3701   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3702   [(set_attr "neon_type" "neon_bp_simple")]
3703 )
3704
3705 (define_insn "neon_vrev16<mode>"
3706   [(set (match_operand:VE 0 "s_register_operand" "=w")
3707         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3708                     (match_operand:SI 2 "immediate_operand" "i")]
3709                    UNSPEC_VREV16))]
3710   "TARGET_NEON"
3711   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3712   [(set_attr "neon_type" "neon_bp_simple")]
3713 )
3714
3715 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3716 ; allocation. For an intrinsic of form:
3717 ;   rD = vbsl_* (rS, rN, rM)
3718 ; We can use any of:
3719 ;   vbsl rS, rN, rM  (if D = S)
3720 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3721 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3722
3723 (define_insn "neon_vbsl<mode>_internal"
3724   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3725         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3726                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3727                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3728                      UNSPEC_VBSL))]
3729   "TARGET_NEON"
3730   "@
3731   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3732   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3733   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3734   [(set_attr "neon_type" "neon_int_1")]
3735 )
3736
3737 (define_expand "neon_vbsl<mode>"
3738   [(set (match_operand:VDQX 0 "s_register_operand" "")
3739         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3740                       (match_operand:VDQX 2 "s_register_operand" "")
3741                       (match_operand:VDQX 3 "s_register_operand" "")]
3742                      UNSPEC_VBSL))]
3743   "TARGET_NEON"
3744 {
3745   /* We can't alias operands together if they have different modes.  */
3746   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3747 })
3748
3749 (define_insn "neon_vshl<mode>"
3750   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3751         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3752                        (match_operand:VDQIX 2 "s_register_operand" "w")
3753                        (match_operand:SI 3 "immediate_operand" "i")]
3754                       UNSPEC_VSHL))]
3755   "TARGET_NEON"
3756   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3757   [(set (attr "neon_type")
3758       (if_then_else (match_test "<Is_d_reg>")
3759                     (const_string "neon_vshl_ddd")
3760                     (const_string "neon_shift_3")))]
3761 )
3762
3763 (define_insn "neon_vqshl<mode>"
3764   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3765         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3766                        (match_operand:VDQIX 2 "s_register_operand" "w")
3767                        (match_operand:SI 3 "immediate_operand" "i")]
3768                       UNSPEC_VQSHL))]
3769   "TARGET_NEON"
3770   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3771   [(set (attr "neon_type")
3772       (if_then_else (match_test "<Is_d_reg>")
3773                     (const_string "neon_shift_2")
3774                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3775 )
3776
3777 (define_insn "neon_vshr_n<mode>"
3778   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3779         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3780                        (match_operand:SI 2 "immediate_operand" "i")
3781                        (match_operand:SI 3 "immediate_operand" "i")]
3782                       UNSPEC_VSHR_N))]
3783   "TARGET_NEON"
3784 {
3785   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3786   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3787 }
3788   [(set_attr "neon_type" "neon_shift_1")]
3789 )
3790
3791 (define_insn "neon_vshrn_n<mode>"
3792   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3793         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3794                             (match_operand:SI 2 "immediate_operand" "i")
3795                             (match_operand:SI 3 "immediate_operand" "i")]
3796                            UNSPEC_VSHRN_N))]
3797   "TARGET_NEON"
3798 {
3799   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3800   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3801 }
3802   [(set_attr "neon_type" "neon_shift_1")]
3803 )
3804
3805 (define_insn "neon_vqshrn_n<mode>"
3806   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3807         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3808                             (match_operand:SI 2 "immediate_operand" "i")
3809                             (match_operand:SI 3 "immediate_operand" "i")]
3810                            UNSPEC_VQSHRN_N))]
3811   "TARGET_NEON"
3812 {
3813   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3814   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3815 }
3816   [(set_attr "neon_type" "neon_shift_2")]
3817 )
3818
3819 (define_insn "neon_vqshrun_n<mode>"
3820   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3821         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3822                             (match_operand:SI 2 "immediate_operand" "i")
3823                             (match_operand:SI 3 "immediate_operand" "i")]
3824                            UNSPEC_VQSHRUN_N))]
3825   "TARGET_NEON"
3826 {
3827   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3828   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3829 }
3830   [(set_attr "neon_type" "neon_shift_2")]
3831 )
3832
3833 (define_insn "neon_vshl_n<mode>"
3834   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3835         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3836                        (match_operand:SI 2 "immediate_operand" "i")
3837                        (match_operand:SI 3 "immediate_operand" "i")]
3838                       UNSPEC_VSHL_N))]
3839   "TARGET_NEON"
3840 {
3841   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3842   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3843 }
3844   [(set_attr "neon_type" "neon_shift_1")]
3845 )
3846
3847 (define_insn "neon_vqshl_n<mode>"
3848   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3849         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3850                        (match_operand:SI 2 "immediate_operand" "i")
3851                        (match_operand:SI 3 "immediate_operand" "i")]
3852                       UNSPEC_VQSHL_N))]
3853   "TARGET_NEON"
3854 {
3855   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3856   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3857 }
3858   [(set_attr "neon_type" "neon_shift_2")]
3859 )
3860
3861 (define_insn "neon_vqshlu_n<mode>"
3862   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3863         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3864                        (match_operand:SI 2 "immediate_operand" "i")
3865                        (match_operand:SI 3 "immediate_operand" "i")]
3866                       UNSPEC_VQSHLU_N))]
3867   "TARGET_NEON"
3868 {
3869   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3870   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3871 }
3872   [(set_attr "neon_type" "neon_shift_2")]
3873 )
3874
3875 (define_insn "neon_vshll_n<mode>"
3876   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3877         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3878                            (match_operand:SI 2 "immediate_operand" "i")
3879                            (match_operand:SI 3 "immediate_operand" "i")]
3880                           UNSPEC_VSHLL_N))]
3881   "TARGET_NEON"
3882 {
3883   /* The boundaries are: 0 < imm <= size.  */
3884   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3885   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3886 }
3887   [(set_attr "neon_type" "neon_shift_1")]
3888 )
3889
3890 (define_insn "neon_vsra_n<mode>"
3891   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3892         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3893                        (match_operand:VDQIX 2 "s_register_operand" "w")
3894                        (match_operand:SI 3 "immediate_operand" "i")
3895                        (match_operand:SI 4 "immediate_operand" "i")]
3896                       UNSPEC_VSRA_N))]
3897   "TARGET_NEON"
3898 {
3899   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3900   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3901 }
3902   [(set_attr "neon_type" "neon_vsra_vrsra")]
3903 )
3904
3905 (define_insn "neon_vsri_n<mode>"
3906   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3907         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3908                        (match_operand:VDQIX 2 "s_register_operand" "w")
3909                        (match_operand:SI 3 "immediate_operand" "i")]
3910                       UNSPEC_VSRI))]
3911   "TARGET_NEON"
3912 {
3913   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3914   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3915 }
3916   [(set (attr "neon_type")
3917       (if_then_else (match_test "<Is_d_reg>")
3918                     (const_string "neon_shift_1")
3919                     (const_string "neon_shift_3")))]
3920 )
3921
3922 (define_insn "neon_vsli_n<mode>"
3923   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3924         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3925                        (match_operand:VDQIX 2 "s_register_operand" "w")
3926                        (match_operand:SI 3 "immediate_operand" "i")]
3927                       UNSPEC_VSLI))]
3928   "TARGET_NEON"
3929 {
3930   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3931   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3932 }
3933   [(set (attr "neon_type")
3934       (if_then_else (match_test "<Is_d_reg>")
3935                     (const_string "neon_shift_1")
3936                     (const_string "neon_shift_3")))]
3937 )
3938
3939 (define_insn "neon_vtbl1v8qi"
3940   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3941         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3942                       (match_operand:V8QI 2 "s_register_operand" "w")]
3943                      UNSPEC_VTBL))]
3944   "TARGET_NEON"
3945   "vtbl.8\t%P0, {%P1}, %P2"
3946   [(set_attr "neon_type" "neon_bp_2cycle")]
3947 )
3948
3949 (define_insn "neon_vtbl2v8qi"
3950   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3951         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3952                       (match_operand:V8QI 2 "s_register_operand" "w")]
3953                      UNSPEC_VTBL))]
3954   "TARGET_NEON"
3955 {
3956   rtx ops[4];
3957   int tabbase = REGNO (operands[1]);
3958
3959   ops[0] = operands[0];
3960   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3961   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3962   ops[3] = operands[2];
3963   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3964
3965   return "";
3966 }
3967   [(set_attr "neon_type" "neon_bp_2cycle")]
3968 )
3969
3970 (define_insn "neon_vtbl3v8qi"
3971   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3972         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3973                       (match_operand:V8QI 2 "s_register_operand" "w")]
3974                      UNSPEC_VTBL))]
3975   "TARGET_NEON"
3976 {
3977   rtx ops[5];
3978   int tabbase = REGNO (operands[1]);
3979
3980   ops[0] = operands[0];
3981   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3982   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3983   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3984   ops[4] = operands[2];
3985   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3986
3987   return "";
3988 }
3989   [(set_attr "neon_type" "neon_bp_3cycle")]
3990 )
3991
3992 (define_insn "neon_vtbl4v8qi"
3993   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3994         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3995                       (match_operand:V8QI 2 "s_register_operand" "w")]
3996                      UNSPEC_VTBL))]
3997   "TARGET_NEON"
3998 {
3999   rtx ops[6];
4000   int tabbase = REGNO (operands[1]);
4001
4002   ops[0] = operands[0];
4003   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4004   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4005   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4006   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4007   ops[5] = operands[2];
4008   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4009
4010   return "";
4011 }
4012   [(set_attr "neon_type" "neon_bp_3cycle")]
4013 )
4014
4015 ;; These three are used by the vec_perm infrastructure for V16QImode.
4016 (define_insn_and_split "neon_vtbl1v16qi"
4017   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4018         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
4019                        (match_operand:V16QI 2 "s_register_operand" "w")]
4020                       UNSPEC_VTBL))]
4021   "TARGET_NEON"
4022   "#"
4023   "&& reload_completed"
4024   [(const_int 0)]
4025 {
4026   rtx op0, op1, op2, part0, part2;
4027   unsigned ofs;
4028
4029   op0 = operands[0];
4030   op1 = gen_lowpart (TImode, operands[1]);
4031   op2 = operands[2];
4032
4033   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4034   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4035   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4036   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4037
4038   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4039   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4040   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4041   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4042   DONE;
4043 })
4044
4045 (define_insn_and_split "neon_vtbl2v16qi"
4046   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4047         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4048                        (match_operand:V16QI 2 "s_register_operand" "w")]
4049                       UNSPEC_VTBL))]
4050   "TARGET_NEON"
4051   "#"
4052   "&& reload_completed"
4053   [(const_int 0)]
4054 {
4055   rtx op0, op1, op2, part0, part2;
4056   unsigned ofs;
4057
4058   op0 = operands[0];
4059   op1 = operands[1];
4060   op2 = operands[2];
4061
4062   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4063   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4064   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4065   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4066
4067   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4068   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4069   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4070   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4071   DONE;
4072 })
4073
4074 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4075 ;; handle quad-word input modes, producing octa-word output modes.  But
4076 ;; that requires us to add support for octa-word vector modes in moves.
4077 ;; That seems overkill for this one use in vec_perm.
4078 (define_insn_and_split "neon_vcombinev16qi"
4079   [(set (match_operand:OI 0 "s_register_operand" "=w")
4080         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4081                     (match_operand:V16QI 2 "s_register_operand" "w")]
4082                    UNSPEC_VCONCAT))]
4083   "TARGET_NEON"
4084   "#"
4085   "&& reload_completed"
4086   [(const_int 0)]
4087 {
4088   neon_split_vcombine (operands);
4089   DONE;
4090 })
4091
4092 (define_insn "neon_vtbx1v8qi"
4093   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4094         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4095                       (match_operand:V8QI 2 "s_register_operand" "w")
4096                       (match_operand:V8QI 3 "s_register_operand" "w")]
4097                      UNSPEC_VTBX))]
4098   "TARGET_NEON"
4099   "vtbx.8\t%P0, {%P2}, %P3"
4100   [(set_attr "neon_type" "neon_bp_2cycle")]
4101 )
4102
4103 (define_insn "neon_vtbx2v8qi"
4104   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4105         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4106                       (match_operand:TI 2 "s_register_operand" "w")
4107                       (match_operand:V8QI 3 "s_register_operand" "w")]
4108                      UNSPEC_VTBX))]
4109   "TARGET_NEON"
4110 {
4111   rtx ops[4];
4112   int tabbase = REGNO (operands[2]);
4113
4114   ops[0] = operands[0];
4115   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4116   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4117   ops[3] = operands[3];
4118   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4119
4120   return "";
4121 }
4122   [(set_attr "neon_type" "neon_bp_2cycle")]
4123 )
4124
4125 (define_insn "neon_vtbx3v8qi"
4126   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4127         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4128                       (match_operand:EI 2 "s_register_operand" "w")
4129                       (match_operand:V8QI 3 "s_register_operand" "w")]
4130                      UNSPEC_VTBX))]
4131   "TARGET_NEON"
4132 {
4133   rtx ops[5];
4134   int tabbase = REGNO (operands[2]);
4135
4136   ops[0] = operands[0];
4137   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4138   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4139   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4140   ops[4] = operands[3];
4141   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4142
4143   return "";
4144 }
4145   [(set_attr "neon_type" "neon_bp_3cycle")]
4146 )
4147
4148 (define_insn "neon_vtbx4v8qi"
4149   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4150         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4151                       (match_operand:OI 2 "s_register_operand" "w")
4152                       (match_operand:V8QI 3 "s_register_operand" "w")]
4153                      UNSPEC_VTBX))]
4154   "TARGET_NEON"
4155 {
4156   rtx ops[6];
4157   int tabbase = REGNO (operands[2]);
4158
4159   ops[0] = operands[0];
4160   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4161   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4162   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4163   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4164   ops[5] = operands[3];
4165   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4166
4167   return "";
4168 }
4169   [(set_attr "neon_type" "neon_bp_3cycle")]
4170 )
4171
4172 (define_insn "neon_vtrn<mode>_internal"
4173   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4174         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4175                       (match_operand:VDQW 2 "s_register_operand" "w")]
4176                      UNSPEC_VTRN1))
4177    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4178          (unspec:VDQW [(match_dup 1) (match_dup 2)]
4179                      UNSPEC_VTRN2))]
4180   "TARGET_NEON"
4181   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4182   [(set (attr "neon_type")
4183       (if_then_else (match_test "<Is_d_reg>")
4184                     (const_string "neon_bp_simple")
4185                     (const_string "neon_bp_3cycle")))]
4186 )
4187
4188 (define_expand "neon_vtrn<mode>"
4189   [(match_operand:SI 0 "s_register_operand" "r")
4190    (match_operand:VDQW 1 "s_register_operand" "w")
4191    (match_operand:VDQW 2 "s_register_operand" "w")]
4192   "TARGET_NEON"
4193 {
4194   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4195                               operands[0], operands[1], operands[2]);
4196   DONE;
4197 })
4198
4199 (define_insn "neon_vzip<mode>_internal"
4200   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4201         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4202                       (match_operand:VDQW 2 "s_register_operand" "w")]
4203                      UNSPEC_VZIP1))
4204    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4205         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4206                      UNSPEC_VZIP2))]
4207   "TARGET_NEON"
4208   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4209   [(set (attr "neon_type")
4210       (if_then_else (match_test "<Is_d_reg>")
4211                     (const_string "neon_bp_simple")
4212                     (const_string "neon_bp_3cycle")))]
4213 )
4214
4215 (define_expand "neon_vzip<mode>"
4216   [(match_operand:SI 0 "s_register_operand" "r")
4217    (match_operand:VDQW 1 "s_register_operand" "w")
4218    (match_operand:VDQW 2 "s_register_operand" "w")]
4219   "TARGET_NEON"
4220 {
4221   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4222                               operands[0], operands[1], operands[2]);
4223   DONE;
4224 })
4225
4226 (define_insn "neon_vuzp<mode>_internal"
4227   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4228         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4229                       (match_operand:VDQW 2 "s_register_operand" "w")]
4230                      UNSPEC_VUZP1))
4231    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4232         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4233                      UNSPEC_VUZP2))]
4234   "TARGET_NEON"
4235   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4236   [(set (attr "neon_type")
4237       (if_then_else (match_test "<Is_d_reg>")
4238                     (const_string "neon_bp_simple")
4239                     (const_string "neon_bp_3cycle")))]
4240 )
4241
4242 (define_expand "neon_vuzp<mode>"
4243   [(match_operand:SI 0 "s_register_operand" "r")
4244    (match_operand:VDQW 1 "s_register_operand" "w")
4245    (match_operand:VDQW 2 "s_register_operand" "w")]
4246   "TARGET_NEON"
4247 {
4248   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4249                               operands[0], operands[1], operands[2]);
4250   DONE;
4251 })
4252
4253 (define_expand "neon_vreinterpretv8qi<mode>"
4254   [(match_operand:V8QI 0 "s_register_operand" "")
4255    (match_operand:VDX 1 "s_register_operand" "")]
4256   "TARGET_NEON"
4257 {
4258   neon_reinterpret (operands[0], operands[1]);
4259   DONE;
4260 })
4261
4262 (define_expand "neon_vreinterpretv4hi<mode>"
4263   [(match_operand:V4HI 0 "s_register_operand" "")
4264    (match_operand:VDX 1 "s_register_operand" "")]
4265   "TARGET_NEON"
4266 {
4267   neon_reinterpret (operands[0], operands[1]);
4268   DONE;
4269 })
4270
4271 (define_expand "neon_vreinterpretv2si<mode>"
4272   [(match_operand:V2SI 0 "s_register_operand" "")
4273    (match_operand:VDX 1 "s_register_operand" "")]
4274   "TARGET_NEON"
4275 {
4276   neon_reinterpret (operands[0], operands[1]);
4277   DONE;
4278 })
4279
4280 (define_expand "neon_vreinterpretv2sf<mode>"
4281   [(match_operand:V2SF 0 "s_register_operand" "")
4282    (match_operand:VDX 1 "s_register_operand" "")]
4283   "TARGET_NEON"
4284 {
4285   neon_reinterpret (operands[0], operands[1]);
4286   DONE;
4287 })
4288
4289 (define_expand "neon_vreinterpretdi<mode>"
4290   [(match_operand:DI 0 "s_register_operand" "")
4291    (match_operand:VDX 1 "s_register_operand" "")]
4292   "TARGET_NEON"
4293 {
4294   neon_reinterpret (operands[0], operands[1]);
4295   DONE;
4296 })
4297
4298 (define_expand "neon_vreinterpretv16qi<mode>"
4299   [(match_operand:V16QI 0 "s_register_operand" "")
4300    (match_operand:VQX 1 "s_register_operand" "")]
4301   "TARGET_NEON"
4302 {
4303   neon_reinterpret (operands[0], operands[1]);
4304   DONE;
4305 })
4306
4307 (define_expand "neon_vreinterpretv8hi<mode>"
4308   [(match_operand:V8HI 0 "s_register_operand" "")
4309    (match_operand:VQX 1 "s_register_operand" "")]
4310   "TARGET_NEON"
4311 {
4312   neon_reinterpret (operands[0], operands[1]);
4313   DONE;
4314 })
4315
4316 (define_expand "neon_vreinterpretv4si<mode>"
4317   [(match_operand:V4SI 0 "s_register_operand" "")
4318    (match_operand:VQX 1 "s_register_operand" "")]
4319   "TARGET_NEON"
4320 {
4321   neon_reinterpret (operands[0], operands[1]);
4322   DONE;
4323 })
4324
4325 (define_expand "neon_vreinterpretv4sf<mode>"
4326   [(match_operand:V4SF 0 "s_register_operand" "")
4327    (match_operand:VQX 1 "s_register_operand" "")]
4328   "TARGET_NEON"
4329 {
4330   neon_reinterpret (operands[0], operands[1]);
4331   DONE;
4332 })
4333
4334 (define_expand "neon_vreinterpretv2di<mode>"
4335   [(match_operand:V2DI 0 "s_register_operand" "")
4336    (match_operand:VQX 1 "s_register_operand" "")]
4337   "TARGET_NEON"
4338 {
4339   neon_reinterpret (operands[0], operands[1]);
4340   DONE;
4341 })
4342
4343 (define_expand "vec_load_lanes<mode><mode>"
4344   [(set (match_operand:VDQX 0 "s_register_operand")
4345         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4346                      UNSPEC_VLD1))]
4347   "TARGET_NEON")
4348
4349 (define_insn "neon_vld1<mode>"
4350   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4351         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4352                     UNSPEC_VLD1))]
4353   "TARGET_NEON"
4354   "vld1.<V_sz_elem>\t%h0, %A1"
4355   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4356 )
4357
4358 (define_insn "neon_vld1_lane<mode>"
4359   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4360         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4361                      (match_operand:VDX 2 "s_register_operand" "0")
4362                      (match_operand:SI 3 "immediate_operand" "i")]
4363                     UNSPEC_VLD1_LANE))]
4364   "TARGET_NEON"
4365 {
4366   HOST_WIDE_INT lane = INTVAL (operands[3]);
4367   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4368   if (lane < 0 || lane >= max)
4369     error ("lane out of range");
4370   if (max == 1)
4371     return "vld1.<V_sz_elem>\t%P0, %A1";
4372   else
4373     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4374 }
4375   [(set (attr "neon_type")
4376       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4377                     (const_string "neon_vld1_1_2_regs")
4378                     (const_string "neon_vld1_vld2_lane")))]
4379 )
4380
4381 (define_insn "neon_vld1_lane<mode>"
4382   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4383         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4384                      (match_operand:VQX 2 "s_register_operand" "0")
4385                      (match_operand:SI 3 "immediate_operand" "i")]
4386                     UNSPEC_VLD1_LANE))]
4387   "TARGET_NEON"
4388 {
4389   HOST_WIDE_INT lane = INTVAL (operands[3]);
4390   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4391   int regno = REGNO (operands[0]);
4392   if (lane < 0 || lane >= max)
4393     error ("lane out of range");
4394   else if (lane >= max / 2)
4395     {
4396       lane -= max / 2;
4397       regno += 2;
4398       operands[3] = GEN_INT (lane);
4399     }
4400   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4401   if (max == 2)
4402     return "vld1.<V_sz_elem>\t%P0, %A1";
4403   else
4404     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4405 }
4406   [(set (attr "neon_type")
4407       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4408                     (const_string "neon_vld1_1_2_regs")
4409                     (const_string "neon_vld1_vld2_lane")))]
4410 )
4411
4412 (define_insn "neon_vld1_dup<mode>"
4413   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4414         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4415   "TARGET_NEON"
4416 {
4417   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4418     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4419   else
4420     return "vld1.<V_sz_elem>\t%h0, %A1";
4421 }
4422   [(set (attr "neon_type")
4423       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4424                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4425                     (const_string "neon_vld1_1_2_regs")))]
4426 )
4427
4428 (define_insn "neon_vld1_dup<mode>"
4429   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4430         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4431   "TARGET_NEON"
4432 {
4433   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4434 }
4435   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4436 )
4437
4438 (define_insn_and_split "neon_vld1_dupv2di"
4439    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4440     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4441    "TARGET_NEON"
4442    "#"
4443    "&& reload_completed"
4444    [(const_int 0)]
4445    {
4446     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4447     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4448     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4449     DONE;
4450     }
4451   [(set_attr "length" "8")
4452    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4453 )
4454
4455 (define_expand "vec_store_lanes<mode><mode>"
4456   [(set (match_operand:VDQX 0 "neon_struct_operand")
4457         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4458                      UNSPEC_VST1))]
4459   "TARGET_NEON")
4460
4461 (define_insn "neon_vst1<mode>"
4462   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4463         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4464                      UNSPEC_VST1))]
4465   "TARGET_NEON"
4466   "vst1.<V_sz_elem>\t%h1, %A0"
4467   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4468
4469 (define_insn "neon_vst1_lane<mode>"
4470   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4471         (unspec:<V_elem>
4472           [(match_operand:VDX 1 "s_register_operand" "w")
4473            (match_operand:SI 2 "immediate_operand" "i")]
4474           UNSPEC_VST1_LANE))]
4475   "TARGET_NEON"
4476 {
4477   HOST_WIDE_INT lane = INTVAL (operands[2]);
4478   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4479   if (lane < 0 || lane >= max)
4480     error ("lane out of range");
4481   if (max == 1)
4482     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4483   else
4484     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4485 }
4486   [(set (attr "neon_type")
4487       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4488                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4489                     (const_string "neon_vst1_vst2_lane")))])
4490
4491 (define_insn "neon_vst1_lane<mode>"
4492   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4493         (unspec:<V_elem>
4494           [(match_operand:VQX 1 "s_register_operand" "w")
4495            (match_operand:SI 2 "immediate_operand" "i")]
4496           UNSPEC_VST1_LANE))]
4497   "TARGET_NEON"
4498 {
4499   HOST_WIDE_INT lane = INTVAL (operands[2]);
4500   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4501   int regno = REGNO (operands[1]);
4502   if (lane < 0 || lane >= max)
4503     error ("lane out of range");
4504   else if (lane >= max / 2)
4505     {
4506       lane -= max / 2;
4507       regno += 2;
4508       operands[2] = GEN_INT (lane);
4509     }
4510   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4511   if (max == 2)
4512     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4513   else
4514     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4515 }
4516   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4517 )
4518
4519 (define_expand "vec_load_lanesti<mode>"
4520   [(set (match_operand:TI 0 "s_register_operand")
4521         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4522                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4523                    UNSPEC_VLD2))]
4524   "TARGET_NEON")
4525
4526 (define_insn "neon_vld2<mode>"
4527   [(set (match_operand:TI 0 "s_register_operand" "=w")
4528         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4529                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4530                    UNSPEC_VLD2))]
4531   "TARGET_NEON"
4532 {
4533   if (<V_sz_elem> == 64)
4534     return "vld1.64\t%h0, %A1";
4535   else
4536     return "vld2.<V_sz_elem>\t%h0, %A1";
4537 }
4538   [(set (attr "neon_type")
4539       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4540                     (const_string "neon_vld1_1_2_regs")
4541                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4542 )
4543
4544 (define_expand "vec_load_lanesoi<mode>"
4545   [(set (match_operand:OI 0 "s_register_operand")
4546         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4547                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4548                    UNSPEC_VLD2))]
4549   "TARGET_NEON")
4550
4551 (define_insn "neon_vld2<mode>"
4552   [(set (match_operand:OI 0 "s_register_operand" "=w")
4553         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4554                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4555                    UNSPEC_VLD2))]
4556   "TARGET_NEON"
4557   "vld2.<V_sz_elem>\t%h0, %A1"
4558   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4559
4560 (define_insn "neon_vld2_lane<mode>"
4561   [(set (match_operand:TI 0 "s_register_operand" "=w")
4562         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4563                     (match_operand:TI 2 "s_register_operand" "0")
4564                     (match_operand:SI 3 "immediate_operand" "i")
4565                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4566                    UNSPEC_VLD2_LANE))]
4567   "TARGET_NEON"
4568 {
4569   HOST_WIDE_INT lane = INTVAL (operands[3]);
4570   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4571   int regno = REGNO (operands[0]);
4572   rtx ops[4];
4573   if (lane < 0 || lane >= max)
4574     error ("lane out of range");
4575   ops[0] = gen_rtx_REG (DImode, regno);
4576   ops[1] = gen_rtx_REG (DImode, regno + 2);
4577   ops[2] = operands[1];
4578   ops[3] = operands[3];
4579   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4580   return "";
4581 }
4582   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4583 )
4584
4585 (define_insn "neon_vld2_lane<mode>"
4586   [(set (match_operand:OI 0 "s_register_operand" "=w")
4587         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4588                     (match_operand:OI 2 "s_register_operand" "0")
4589                     (match_operand:SI 3 "immediate_operand" "i")
4590                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4591                    UNSPEC_VLD2_LANE))]
4592   "TARGET_NEON"
4593 {
4594   HOST_WIDE_INT lane = INTVAL (operands[3]);
4595   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4596   int regno = REGNO (operands[0]);
4597   rtx ops[4];
4598   if (lane < 0 || lane >= max)
4599     error ("lane out of range");
4600   else if (lane >= max / 2)
4601     {
4602       lane -= max / 2;
4603       regno += 2;
4604     }
4605   ops[0] = gen_rtx_REG (DImode, regno);
4606   ops[1] = gen_rtx_REG (DImode, regno + 4);
4607   ops[2] = operands[1];
4608   ops[3] = GEN_INT (lane);
4609   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4610   return "";
4611 }
4612   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4613 )
4614
4615 (define_insn "neon_vld2_dup<mode>"
4616   [(set (match_operand:TI 0 "s_register_operand" "=w")
4617         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4618                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4619                    UNSPEC_VLD2_DUP))]
4620   "TARGET_NEON"
4621 {
4622   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4623     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4624   else
4625     return "vld1.<V_sz_elem>\t%h0, %A1";
4626 }
4627   [(set (attr "neon_type")
4628       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4629                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4630                     (const_string "neon_vld1_1_2_regs")))]
4631 )
4632
4633 (define_expand "vec_store_lanesti<mode>"
4634   [(set (match_operand:TI 0 "neon_struct_operand")
4635         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4636                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4637                    UNSPEC_VST2))]
4638   "TARGET_NEON")
4639
4640 (define_insn "neon_vst2<mode>"
4641   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4642         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4643                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4644                    UNSPEC_VST2))]
4645   "TARGET_NEON"
4646 {
4647   if (<V_sz_elem> == 64)
4648     return "vst1.64\t%h1, %A0";
4649   else
4650     return "vst2.<V_sz_elem>\t%h1, %A0";
4651 }
4652   [(set (attr "neon_type")
4653       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4654                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4655                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4656 )
4657
4658 (define_expand "vec_store_lanesoi<mode>"
4659   [(set (match_operand:OI 0 "neon_struct_operand")
4660         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4661                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4662                    UNSPEC_VST2))]
4663   "TARGET_NEON")
4664
4665 (define_insn "neon_vst2<mode>"
4666   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4667         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4668                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4669                    UNSPEC_VST2))]
4670   "TARGET_NEON"
4671   "vst2.<V_sz_elem>\t%h1, %A0"
4672   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4673 )
4674
4675 (define_insn "neon_vst2_lane<mode>"
4676   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4677         (unspec:<V_two_elem>
4678           [(match_operand:TI 1 "s_register_operand" "w")
4679            (match_operand:SI 2 "immediate_operand" "i")
4680            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4681           UNSPEC_VST2_LANE))]
4682   "TARGET_NEON"
4683 {
4684   HOST_WIDE_INT lane = INTVAL (operands[2]);
4685   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4686   int regno = REGNO (operands[1]);
4687   rtx ops[4];
4688   if (lane < 0 || lane >= max)
4689     error ("lane out of range");
4690   ops[0] = operands[0];
4691   ops[1] = gen_rtx_REG (DImode, regno);
4692   ops[2] = gen_rtx_REG (DImode, regno + 2);
4693   ops[3] = operands[2];
4694   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4695   return "";
4696 }
4697   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4698 )
4699
4700 (define_insn "neon_vst2_lane<mode>"
4701   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4702         (unspec:<V_two_elem>
4703            [(match_operand:OI 1 "s_register_operand" "w")
4704             (match_operand:SI 2 "immediate_operand" "i")
4705             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4706            UNSPEC_VST2_LANE))]
4707   "TARGET_NEON"
4708 {
4709   HOST_WIDE_INT lane = INTVAL (operands[2]);
4710   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4711   int regno = REGNO (operands[1]);
4712   rtx ops[4];
4713   if (lane < 0 || lane >= max)
4714     error ("lane out of range");
4715   else if (lane >= max / 2)
4716     {
4717       lane -= max / 2;
4718       regno += 2;
4719     }
4720   ops[0] = operands[0];
4721   ops[1] = gen_rtx_REG (DImode, regno);
4722   ops[2] = gen_rtx_REG (DImode, regno + 4);
4723   ops[3] = GEN_INT (lane);
4724   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4725   return "";
4726 }
4727   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4728 )
4729
4730 (define_expand "vec_load_lanesei<mode>"
4731   [(set (match_operand:EI 0 "s_register_operand")
4732         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4733                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4734                    UNSPEC_VLD3))]
4735   "TARGET_NEON")
4736
4737 (define_insn "neon_vld3<mode>"
4738   [(set (match_operand:EI 0 "s_register_operand" "=w")
4739         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4740                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4741                    UNSPEC_VLD3))]
4742   "TARGET_NEON"
4743 {
4744   if (<V_sz_elem> == 64)
4745     return "vld1.64\t%h0, %A1";
4746   else
4747     return "vld3.<V_sz_elem>\t%h0, %A1";
4748 }
4749   [(set (attr "neon_type")
4750       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4751                     (const_string "neon_vld1_1_2_regs")
4752                     (const_string "neon_vld3_vld4")))]
4753 )
4754
4755 (define_expand "vec_load_lanesci<mode>"
4756   [(match_operand:CI 0 "s_register_operand")
4757    (match_operand:CI 1 "neon_struct_operand")
4758    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4759   "TARGET_NEON"
4760 {
4761   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4762   DONE;
4763 })
4764
4765 (define_expand "neon_vld3<mode>"
4766   [(match_operand:CI 0 "s_register_operand")
4767    (match_operand:CI 1 "neon_struct_operand")
4768    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4769   "TARGET_NEON"
4770 {
4771   rtx mem;
4772
4773   mem = adjust_address (operands[1], EImode, 0);
4774   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4775   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4776   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4777   DONE;
4778 })
4779
4780 (define_insn "neon_vld3qa<mode>"
4781   [(set (match_operand:CI 0 "s_register_operand" "=w")
4782         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4783                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4784                    UNSPEC_VLD3A))]
4785   "TARGET_NEON"
4786 {
4787   int regno = REGNO (operands[0]);
4788   rtx ops[4];
4789   ops[0] = gen_rtx_REG (DImode, regno);
4790   ops[1] = gen_rtx_REG (DImode, regno + 4);
4791   ops[2] = gen_rtx_REG (DImode, regno + 8);
4792   ops[3] = operands[1];
4793   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4794   return "";
4795 }
4796   [(set_attr "neon_type" "neon_vld3_vld4")]
4797 )
4798
4799 (define_insn "neon_vld3qb<mode>"
4800   [(set (match_operand:CI 0 "s_register_operand" "=w")
4801         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4802                     (match_operand:CI 2 "s_register_operand" "0")
4803                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4804                    UNSPEC_VLD3B))]
4805   "TARGET_NEON"
4806 {
4807   int regno = REGNO (operands[0]);
4808   rtx ops[4];
4809   ops[0] = gen_rtx_REG (DImode, regno + 2);
4810   ops[1] = gen_rtx_REG (DImode, regno + 6);
4811   ops[2] = gen_rtx_REG (DImode, regno + 10);
4812   ops[3] = operands[1];
4813   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4814   return "";
4815 }
4816   [(set_attr "neon_type" "neon_vld3_vld4")]
4817 )
4818
4819 (define_insn "neon_vld3_lane<mode>"
4820   [(set (match_operand:EI 0 "s_register_operand" "=w")
4821         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4822                     (match_operand:EI 2 "s_register_operand" "0")
4823                     (match_operand:SI 3 "immediate_operand" "i")
4824                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4825                    UNSPEC_VLD3_LANE))]
4826   "TARGET_NEON"
4827 {
4828   HOST_WIDE_INT lane = INTVAL (operands[3]);
4829   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4830   int regno = REGNO (operands[0]);
4831   rtx ops[5];
4832   if (lane < 0 || lane >= max)
4833     error ("lane out of range");
4834   ops[0] = gen_rtx_REG (DImode, regno);
4835   ops[1] = gen_rtx_REG (DImode, regno + 2);
4836   ops[2] = gen_rtx_REG (DImode, regno + 4);
4837   ops[3] = operands[1];
4838   ops[4] = operands[3];
4839   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4840                    ops);
4841   return "";
4842 }
4843   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4844 )
4845
4846 (define_insn "neon_vld3_lane<mode>"
4847   [(set (match_operand:CI 0 "s_register_operand" "=w")
4848         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4849                     (match_operand:CI 2 "s_register_operand" "0")
4850                     (match_operand:SI 3 "immediate_operand" "i")
4851                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4852                    UNSPEC_VLD3_LANE))]
4853   "TARGET_NEON"
4854 {
4855   HOST_WIDE_INT lane = INTVAL (operands[3]);
4856   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4857   int regno = REGNO (operands[0]);
4858   rtx ops[5];
4859   if (lane < 0 || lane >= max)
4860     error ("lane out of range");
4861   else if (lane >= max / 2)
4862     {
4863       lane -= max / 2;
4864       regno += 2;
4865     }
4866   ops[0] = gen_rtx_REG (DImode, regno);
4867   ops[1] = gen_rtx_REG (DImode, regno + 4);
4868   ops[2] = gen_rtx_REG (DImode, regno + 8);
4869   ops[3] = operands[1];
4870   ops[4] = GEN_INT (lane);
4871   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4872                    ops);
4873   return "";
4874 }
4875   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4876 )
4877
4878 (define_insn "neon_vld3_dup<mode>"
4879   [(set (match_operand:EI 0 "s_register_operand" "=w")
4880         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4881                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4882                    UNSPEC_VLD3_DUP))]
4883   "TARGET_NEON"
4884 {
4885   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4886     {
4887       int regno = REGNO (operands[0]);
4888       rtx ops[4];
4889       ops[0] = gen_rtx_REG (DImode, regno);
4890       ops[1] = gen_rtx_REG (DImode, regno + 2);
4891       ops[2] = gen_rtx_REG (DImode, regno + 4);
4892       ops[3] = operands[1];
4893       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
4894       return "";
4895     }
4896   else
4897     return "vld1.<V_sz_elem>\t%h0, %A1";
4898 }
4899   [(set (attr "neon_type")
4900       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4901                     (const_string "neon_vld3_vld4_all_lanes")
4902                     (const_string "neon_vld1_1_2_regs")))])
4903
4904 (define_expand "vec_store_lanesei<mode>"
4905   [(set (match_operand:EI 0 "neon_struct_operand")
4906         (unspec:EI [(match_operand:EI 1 "s_register_operand")
4907                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4908                    UNSPEC_VST3))]
4909   "TARGET_NEON")
4910
4911 (define_insn "neon_vst3<mode>"
4912   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4913         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4914                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4915                    UNSPEC_VST3))]
4916   "TARGET_NEON"
4917 {
4918   if (<V_sz_elem> == 64)
4919     return "vst1.64\t%h1, %A0";
4920   else
4921     return "vst3.<V_sz_elem>\t%h1, %A0";
4922 }
4923   [(set (attr "neon_type")
4924       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4925                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4926                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
4927
4928 (define_expand "vec_store_lanesci<mode>"
4929   [(match_operand:CI 0 "neon_struct_operand")
4930    (match_operand:CI 1 "s_register_operand")
4931    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4932   "TARGET_NEON"
4933 {
4934   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4935   DONE;
4936 })
4937
4938 (define_expand "neon_vst3<mode>"
4939   [(match_operand:CI 0 "neon_struct_operand")
4940    (match_operand:CI 1 "s_register_operand")
4941    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4942   "TARGET_NEON"
4943 {
4944   rtx mem;
4945
4946   mem = adjust_address (operands[0], EImode, 0);
4947   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4948   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4949   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
4950   DONE;
4951 })
4952
4953 (define_insn "neon_vst3qa<mode>"
4954   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4955         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4956                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4957                    UNSPEC_VST3A))]
4958   "TARGET_NEON"
4959 {
4960   int regno = REGNO (operands[1]);
4961   rtx ops[4];
4962   ops[0] = operands[0];
4963   ops[1] = gen_rtx_REG (DImode, regno);
4964   ops[2] = gen_rtx_REG (DImode, regno + 4);
4965   ops[3] = gen_rtx_REG (DImode, regno + 8);
4966   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4967   return "";
4968 }
4969   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4970 )
4971
4972 (define_insn "neon_vst3qb<mode>"
4973   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4974         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4975                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4976                    UNSPEC_VST3B))]
4977   "TARGET_NEON"
4978 {
4979   int regno = REGNO (operands[1]);
4980   rtx ops[4];
4981   ops[0] = operands[0];
4982   ops[1] = gen_rtx_REG (DImode, regno + 2);
4983   ops[2] = gen_rtx_REG (DImode, regno + 6);
4984   ops[3] = gen_rtx_REG (DImode, regno + 10);
4985   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4986   return "";
4987 }
4988   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4989 )
4990
4991 (define_insn "neon_vst3_lane<mode>"
4992   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4993         (unspec:<V_three_elem>
4994            [(match_operand:EI 1 "s_register_operand" "w")
4995             (match_operand:SI 2 "immediate_operand" "i")
4996             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4997            UNSPEC_VST3_LANE))]
4998   "TARGET_NEON"
4999 {
5000   HOST_WIDE_INT lane = INTVAL (operands[2]);
5001   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5002   int regno = REGNO (operands[1]);
5003   rtx ops[5];
5004   if (lane < 0 || lane >= max)
5005     error ("lane out of range");
5006   ops[0] = operands[0];
5007   ops[1] = gen_rtx_REG (DImode, regno);
5008   ops[2] = gen_rtx_REG (DImode, regno + 2);
5009   ops[3] = gen_rtx_REG (DImode, regno + 4);
5010   ops[4] = operands[2];
5011   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5012                    ops);
5013   return "";
5014 }
5015   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5016 )
5017
5018 (define_insn "neon_vst3_lane<mode>"
5019   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5020         (unspec:<V_three_elem>
5021            [(match_operand:CI 1 "s_register_operand" "w")
5022             (match_operand:SI 2 "immediate_operand" "i")
5023             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5024            UNSPEC_VST3_LANE))]
5025   "TARGET_NEON"
5026 {
5027   HOST_WIDE_INT lane = INTVAL (operands[2]);
5028   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5029   int regno = REGNO (operands[1]);
5030   rtx ops[5];
5031   if (lane < 0 || lane >= max)
5032     error ("lane out of range");
5033   else if (lane >= max / 2)
5034     {
5035       lane -= max / 2;
5036       regno += 2;
5037     }
5038   ops[0] = operands[0];
5039   ops[1] = gen_rtx_REG (DImode, regno);
5040   ops[2] = gen_rtx_REG (DImode, regno + 4);
5041   ops[3] = gen_rtx_REG (DImode, regno + 8);
5042   ops[4] = GEN_INT (lane);
5043   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5044                    ops);
5045   return "";
5046 }
5047 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5048
5049 (define_expand "vec_load_lanesoi<mode>"
5050   [(set (match_operand:OI 0 "s_register_operand")
5051         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5052                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5053                    UNSPEC_VLD4))]
5054   "TARGET_NEON")
5055
5056 (define_insn "neon_vld4<mode>"
5057   [(set (match_operand:OI 0 "s_register_operand" "=w")
5058         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5059                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5060                    UNSPEC_VLD4))]
5061   "TARGET_NEON"
5062 {
5063   if (<V_sz_elem> == 64)
5064     return "vld1.64\t%h0, %A1";
5065   else
5066     return "vld4.<V_sz_elem>\t%h0, %A1";
5067 }
5068   [(set (attr "neon_type")
5069       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5070                     (const_string "neon_vld1_1_2_regs")
5071                     (const_string "neon_vld3_vld4")))]
5072 )
5073
5074 (define_expand "vec_load_lanesxi<mode>"
5075   [(match_operand:XI 0 "s_register_operand")
5076    (match_operand:XI 1 "neon_struct_operand")
5077    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5078   "TARGET_NEON"
5079 {
5080   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5081   DONE;
5082 })
5083
5084 (define_expand "neon_vld4<mode>"
5085   [(match_operand:XI 0 "s_register_operand")
5086    (match_operand:XI 1 "neon_struct_operand")
5087    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5088   "TARGET_NEON"
5089 {
5090   rtx mem;
5091
5092   mem = adjust_address (operands[1], OImode, 0);
5093   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5094   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5095   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5096   DONE;
5097 })
5098
5099 (define_insn "neon_vld4qa<mode>"
5100   [(set (match_operand:XI 0 "s_register_operand" "=w")
5101         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5102                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5103                    UNSPEC_VLD4A))]
5104   "TARGET_NEON"
5105 {
5106   int regno = REGNO (operands[0]);
5107   rtx ops[5];
5108   ops[0] = gen_rtx_REG (DImode, regno);
5109   ops[1] = gen_rtx_REG (DImode, regno + 4);
5110   ops[2] = gen_rtx_REG (DImode, regno + 8);
5111   ops[3] = gen_rtx_REG (DImode, regno + 12);
5112   ops[4] = operands[1];
5113   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5114   return "";
5115 }
5116   [(set_attr "neon_type" "neon_vld3_vld4")]
5117 )
5118
5119 (define_insn "neon_vld4qb<mode>"
5120   [(set (match_operand:XI 0 "s_register_operand" "=w")
5121         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5122                     (match_operand:XI 2 "s_register_operand" "0")
5123                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5124                    UNSPEC_VLD4B))]
5125   "TARGET_NEON"
5126 {
5127   int regno = REGNO (operands[0]);
5128   rtx ops[5];
5129   ops[0] = gen_rtx_REG (DImode, regno + 2);
5130   ops[1] = gen_rtx_REG (DImode, regno + 6);
5131   ops[2] = gen_rtx_REG (DImode, regno + 10);
5132   ops[3] = gen_rtx_REG (DImode, regno + 14);
5133   ops[4] = operands[1];
5134   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5135   return "";
5136 }
5137   [(set_attr "neon_type" "neon_vld3_vld4")]
5138 )
5139
5140 (define_insn "neon_vld4_lane<mode>"
5141   [(set (match_operand:OI 0 "s_register_operand" "=w")
5142         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5143                     (match_operand:OI 2 "s_register_operand" "0")
5144                     (match_operand:SI 3 "immediate_operand" "i")
5145                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5146                    UNSPEC_VLD4_LANE))]
5147   "TARGET_NEON"
5148 {
5149   HOST_WIDE_INT lane = INTVAL (operands[3]);
5150   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5151   int regno = REGNO (operands[0]);
5152   rtx ops[6];
5153   if (lane < 0 || lane >= max)
5154     error ("lane out of range");
5155   ops[0] = gen_rtx_REG (DImode, regno);
5156   ops[1] = gen_rtx_REG (DImode, regno + 2);
5157   ops[2] = gen_rtx_REG (DImode, regno + 4);
5158   ops[3] = gen_rtx_REG (DImode, regno + 6);
5159   ops[4] = operands[1];
5160   ops[5] = operands[3];
5161   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5162                    ops);
5163   return "";
5164 }
5165   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5166 )
5167
5168 (define_insn "neon_vld4_lane<mode>"
5169   [(set (match_operand:XI 0 "s_register_operand" "=w")
5170         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5171                     (match_operand:XI 2 "s_register_operand" "0")
5172                     (match_operand:SI 3 "immediate_operand" "i")
5173                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5174                    UNSPEC_VLD4_LANE))]
5175   "TARGET_NEON"
5176 {
5177   HOST_WIDE_INT lane = INTVAL (operands[3]);
5178   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5179   int regno = REGNO (operands[0]);
5180   rtx ops[6];
5181   if (lane < 0 || lane >= max)
5182     error ("lane out of range");
5183   else if (lane >= max / 2)
5184     {
5185       lane -= max / 2;
5186       regno += 2;
5187     }
5188   ops[0] = gen_rtx_REG (DImode, regno);
5189   ops[1] = gen_rtx_REG (DImode, regno + 4);
5190   ops[2] = gen_rtx_REG (DImode, regno + 8);
5191   ops[3] = gen_rtx_REG (DImode, regno + 12);
5192   ops[4] = operands[1];
5193   ops[5] = GEN_INT (lane);
5194   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5195                    ops);
5196   return "";
5197 }
5198   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5199 )
5200
5201 (define_insn "neon_vld4_dup<mode>"
5202   [(set (match_operand:OI 0 "s_register_operand" "=w")
5203         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5204                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5205                    UNSPEC_VLD4_DUP))]
5206   "TARGET_NEON"
5207 {
5208   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5209     {
5210       int regno = REGNO (operands[0]);
5211       rtx ops[5];
5212       ops[0] = gen_rtx_REG (DImode, regno);
5213       ops[1] = gen_rtx_REG (DImode, regno + 2);
5214       ops[2] = gen_rtx_REG (DImode, regno + 4);
5215       ops[3] = gen_rtx_REG (DImode, regno + 6);
5216       ops[4] = operands[1];
5217       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5218                        ops);
5219       return "";
5220     }
5221   else
5222     return "vld1.<V_sz_elem>\t%h0, %A1";
5223 }
5224   [(set (attr "neon_type")
5225       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5226                     (const_string "neon_vld3_vld4_all_lanes")
5227                     (const_string "neon_vld1_1_2_regs")))]
5228 )
5229
5230 (define_expand "vec_store_lanesoi<mode>"
5231   [(set (match_operand:OI 0 "neon_struct_operand")
5232         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5233                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5234                    UNSPEC_VST4))]
5235   "TARGET_NEON")
5236
5237 (define_insn "neon_vst4<mode>"
5238   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5239         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5240                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5241                    UNSPEC_VST4))]
5242   "TARGET_NEON"
5243 {
5244   if (<V_sz_elem> == 64)
5245     return "vst1.64\t%h1, %A0";
5246   else
5247     return "vst4.<V_sz_elem>\t%h1, %A0";
5248 }
5249   [(set (attr "neon_type")
5250       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5251                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5252                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5253 )
5254
5255 (define_expand "vec_store_lanesxi<mode>"
5256   [(match_operand:XI 0 "neon_struct_operand")
5257    (match_operand:XI 1 "s_register_operand")
5258    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5259   "TARGET_NEON"
5260 {
5261   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5262   DONE;
5263 })
5264
5265 (define_expand "neon_vst4<mode>"
5266   [(match_operand:XI 0 "neon_struct_operand")
5267    (match_operand:XI 1 "s_register_operand")
5268    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5269   "TARGET_NEON"
5270 {
5271   rtx mem;
5272
5273   mem = adjust_address (operands[0], OImode, 0);
5274   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5275   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5276   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5277   DONE;
5278 })
5279
5280 (define_insn "neon_vst4qa<mode>"
5281   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5282         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5283                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5284                    UNSPEC_VST4A))]
5285   "TARGET_NEON"
5286 {
5287   int regno = REGNO (operands[1]);
5288   rtx ops[5];
5289   ops[0] = operands[0];
5290   ops[1] = gen_rtx_REG (DImode, regno);
5291   ops[2] = gen_rtx_REG (DImode, regno + 4);
5292   ops[3] = gen_rtx_REG (DImode, regno + 8);
5293   ops[4] = gen_rtx_REG (DImode, regno + 12);
5294   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5295   return "";
5296 }
5297   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5298 )
5299
5300 (define_insn "neon_vst4qb<mode>"
5301   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5302         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5303                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5304                    UNSPEC_VST4B))]
5305   "TARGET_NEON"
5306 {
5307   int regno = REGNO (operands[1]);
5308   rtx ops[5];
5309   ops[0] = operands[0];
5310   ops[1] = gen_rtx_REG (DImode, regno + 2);
5311   ops[2] = gen_rtx_REG (DImode, regno + 6);
5312   ops[3] = gen_rtx_REG (DImode, regno + 10);
5313   ops[4] = gen_rtx_REG (DImode, regno + 14);
5314   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5315   return "";
5316 }
5317   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5318 )
5319
5320 (define_insn "neon_vst4_lane<mode>"
5321   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5322         (unspec:<V_four_elem>
5323            [(match_operand:OI 1 "s_register_operand" "w")
5324             (match_operand:SI 2 "immediate_operand" "i")
5325             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5326            UNSPEC_VST4_LANE))]
5327   "TARGET_NEON"
5328 {
5329   HOST_WIDE_INT lane = INTVAL (operands[2]);
5330   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5331   int regno = REGNO (operands[1]);
5332   rtx ops[6];
5333   if (lane < 0 || lane >= max)
5334     error ("lane out of range");
5335   ops[0] = operands[0];
5336   ops[1] = gen_rtx_REG (DImode, regno);
5337   ops[2] = gen_rtx_REG (DImode, regno + 2);
5338   ops[3] = gen_rtx_REG (DImode, regno + 4);
5339   ops[4] = gen_rtx_REG (DImode, regno + 6);
5340   ops[5] = operands[2];
5341   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5342                    ops);
5343   return "";
5344 }
5345   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5346 )
5347
5348 (define_insn "neon_vst4_lane<mode>"
5349   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5350         (unspec:<V_four_elem>
5351            [(match_operand:XI 1 "s_register_operand" "w")
5352             (match_operand:SI 2 "immediate_operand" "i")
5353             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5354            UNSPEC_VST4_LANE))]
5355   "TARGET_NEON"
5356 {
5357   HOST_WIDE_INT lane = INTVAL (operands[2]);
5358   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5359   int regno = REGNO (operands[1]);
5360   rtx ops[6];
5361   if (lane < 0 || lane >= max)
5362     error ("lane out of range");
5363   else if (lane >= max / 2)
5364     {
5365       lane -= max / 2;
5366       regno += 2;
5367     }
5368   ops[0] = operands[0];
5369   ops[1] = gen_rtx_REG (DImode, regno);
5370   ops[2] = gen_rtx_REG (DImode, regno + 4);
5371   ops[3] = gen_rtx_REG (DImode, regno + 8);
5372   ops[4] = gen_rtx_REG (DImode, regno + 12);
5373   ops[5] = GEN_INT (lane);
5374   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5375                    ops);
5376   return "";
5377 }
5378   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5379 )
5380
5381 (define_expand "neon_vand<mode>"
5382   [(match_operand:VDQX 0 "s_register_operand" "")
5383    (match_operand:VDQX 1 "s_register_operand" "")
5384    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5385    (match_operand:SI 3 "immediate_operand" "")]
5386   "TARGET_NEON"
5387 {
5388   emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5389   DONE;
5390 })
5391
5392 (define_expand "neon_vorr<mode>"
5393   [(match_operand:VDQX 0 "s_register_operand" "")
5394    (match_operand:VDQX 1 "s_register_operand" "")
5395    (match_operand:VDQX 2 "neon_logic_op2" "")
5396    (match_operand:SI 3 "immediate_operand" "")]
5397   "TARGET_NEON"
5398 {
5399   emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5400   DONE;
5401 })
5402
5403 (define_expand "neon_veor<mode>"
5404   [(match_operand:VDQX 0 "s_register_operand" "")
5405    (match_operand:VDQX 1 "s_register_operand" "")
5406    (match_operand:VDQX 2 "s_register_operand" "")
5407    (match_operand:SI 3 "immediate_operand" "")]
5408   "TARGET_NEON"
5409 {
5410   emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5411   DONE;
5412 })
5413
5414 (define_expand "neon_vbic<mode>"
5415   [(match_operand:VDQX 0 "s_register_operand" "")
5416    (match_operand:VDQX 1 "s_register_operand" "")
5417    (match_operand:VDQX 2 "neon_logic_op2" "")
5418    (match_operand:SI 3 "immediate_operand" "")]
5419   "TARGET_NEON"
5420 {
5421   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5422   DONE;
5423 })
5424
5425 (define_expand "neon_vorn<mode>"
5426   [(match_operand:VDQX 0 "s_register_operand" "")
5427    (match_operand:VDQX 1 "s_register_operand" "")
5428    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5429    (match_operand:SI 3 "immediate_operand" "")]
5430   "TARGET_NEON"
5431 {
5432   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5433   DONE;
5434 })
5435
5436 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5437   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5438         (SE:<V_unpack> (vec_select:<V_HALF>
5439                           (match_operand:VU 1 "register_operand" "w")
5440                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5441   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5442   "vmovl.<US><V_sz_elem> %q0, %e1"
5443   [(set_attr "neon_type" "neon_shift_1")]
5444 )
5445
5446 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5447   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5448         (SE:<V_unpack> (vec_select:<V_HALF>
5449                           (match_operand:VU 1 "register_operand" "w")
5450                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5451   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5452   "vmovl.<US><V_sz_elem> %q0, %f1"
5453   [(set_attr "neon_type" "neon_shift_1")]
5454 )
5455
5456 (define_expand "vec_unpack<US>_hi_<mode>"
5457   [(match_operand:<V_unpack> 0 "register_operand" "")
5458    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5459  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5460   {
5461    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5462    rtx t1;
5463    int i;
5464    for (i = 0; i < (<V_mode_nunits>/2); i++)
5465      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5466
5467    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5468    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5469                                                  operands[1],
5470                                                  t1));
5471    DONE;
5472   }
5473 )
5474
5475 (define_expand "vec_unpack<US>_lo_<mode>"
5476   [(match_operand:<V_unpack> 0 "register_operand" "")
5477    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5478  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5479   {
5480    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5481    rtx t1;
5482    int i;
5483    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5484      RTVEC_ELT (v, i) = GEN_INT (i);
5485    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5486    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5487                                                  operands[1],
5488                                                  t1));
5489    DONE;
5490   }
5491 )
5492
5493 (define_insn "neon_vec_<US>mult_lo_<mode>"
5494  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5495        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5496                            (match_operand:VU 1 "register_operand" "w")
5497                            (match_operand:VU 2 "vect_par_constant_low" "")))
5498                         (SE:<V_unpack> (vec_select:<V_HALF>
5499                            (match_operand:VU 3 "register_operand" "w")
5500                            (match_dup 2)))))]
5501   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5502   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5503   [(set_attr "neon_type" "neon_shift_1")]
5504 )
5505
5506 (define_expand "vec_widen_<US>mult_lo_<mode>"
5507   [(match_operand:<V_unpack> 0 "register_operand" "")
5508    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5509    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5510  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5511  {
5512    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5513    rtx t1;
5514    int i;
5515    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5516      RTVEC_ELT (v, i) = GEN_INT (i);
5517    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5518
5519    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5520                                                operands[1],
5521                                                t1,
5522                                                operands[2]));
5523    DONE;
5524  }
5525 )
5526
5527 (define_insn "neon_vec_<US>mult_hi_<mode>"
5528  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5529       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5530                             (match_operand:VU 1 "register_operand" "w")
5531                             (match_operand:VU 2 "vect_par_constant_high" "")))
5532                        (SE:<V_unpack> (vec_select:<V_HALF>
5533                             (match_operand:VU 3 "register_operand" "w")
5534                             (match_dup 2)))))]
5535   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5536   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5537   [(set_attr "neon_type" "neon_shift_1")]
5538 )
5539
5540 (define_expand "vec_widen_<US>mult_hi_<mode>"
5541   [(match_operand:<V_unpack> 0 "register_operand" "")
5542    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5543    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5544  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5545  {
5546    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5547    rtx t1;
5548    int i;
5549    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5550      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5551    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5552
5553    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5554                                                operands[1],
5555                                                t1,
5556                                                operands[2]));
5557    DONE;
5558
5559  }
5560 )
5561
5562 (define_insn "neon_vec_<US>shiftl_<mode>"
5563  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5564        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5565        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5566   "TARGET_NEON"
5567 {
5568   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5569 }
5570   [(set_attr "neon_type" "neon_shift_1")]
5571 )
5572
5573 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5574   [(match_operand:<V_unpack> 0 "register_operand" "")
5575    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5576    (match_operand:SI 2 "immediate_operand" "i")]
5577  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5578  {
5579   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5580                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5581                 operands[2]));
5582    DONE;
5583  }
5584 )
5585
5586 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5587   [(match_operand:<V_unpack> 0 "register_operand" "")
5588    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5589    (match_operand:SI 2 "immediate_operand" "i")]
5590  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5591  {
5592   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5593                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5594                                      GET_MODE_SIZE (<V_HALF>mode)),
5595                 operands[2]));
5596    DONE;
5597  }
5598 )
5599
5600 ;; Vectorize for non-neon-quad case
5601 (define_insn "neon_unpack<US>_<mode>"
5602  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5603        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5604  "TARGET_NEON"
5605  "vmovl.<US><V_sz_elem> %q0, %P1"
5606   [(set_attr "neon_type" "neon_shift_1")]
5607 )
5608
5609 (define_expand "vec_unpack<US>_lo_<mode>"
5610  [(match_operand:<V_double_width> 0 "register_operand" "")
5611   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5612  "TARGET_NEON"
5613 {
5614   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5615   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5616   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5617
5618   DONE;
5619 }
5620 )
5621
5622 (define_expand "vec_unpack<US>_hi_<mode>"
5623  [(match_operand:<V_double_width> 0 "register_operand" "")
5624   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5625  "TARGET_NEON"
5626 {
5627   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5628   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5629   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5630
5631   DONE;
5632 }
5633 )
5634
5635 (define_insn "neon_vec_<US>mult_<mode>"
5636  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5637        (mult:<V_widen> (SE:<V_widen>
5638                            (match_operand:VDI 1 "register_operand" "w"))
5639                        (SE:<V_widen>
5640                            (match_operand:VDI 2 "register_operand" "w"))))]
5641   "TARGET_NEON"
5642   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5643   [(set_attr "neon_type" "neon_shift_1")]
5644 )
5645
5646 (define_expand "vec_widen_<US>mult_hi_<mode>"
5647   [(match_operand:<V_double_width> 0 "register_operand" "")
5648    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5649    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5650  "TARGET_NEON"
5651  {
5652    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5653    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5654    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5655
5656    DONE;
5657
5658  }
5659 )
5660
5661 (define_expand "vec_widen_<US>mult_lo_<mode>"
5662   [(match_operand:<V_double_width> 0 "register_operand" "")
5663    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5664    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5665  "TARGET_NEON"
5666  {
5667    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5668    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5669    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5670
5671    DONE;
5672
5673  }
5674 )
5675
5676 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5677  [(match_operand:<V_double_width> 0 "register_operand" "")
5678    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5679    (match_operand:SI 2 "immediate_operand" "i")]
5680  "TARGET_NEON"
5681  {
5682    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5683    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5684    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5685
5686    DONE;
5687  }
5688 )
5689
5690 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5691   [(match_operand:<V_double_width> 0 "register_operand" "")
5692    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5693    (match_operand:SI 2 "immediate_operand" "i")]
5694  "TARGET_NEON"
5695  {
5696    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5697    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5698    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5699
5700    DONE;
5701  }
5702 )
5703
5704 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5705 ; because the ordering of vector elements in Q registers is different from what
5706 ; the semantics of the instructions require.
5707
5708 (define_insn "vec_pack_trunc_<mode>"
5709  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5710        (vec_concat:<V_narrow_pack>
5711                 (truncate:<V_narrow>
5712                         (match_operand:VN 1 "register_operand" "w"))
5713                 (truncate:<V_narrow>
5714                         (match_operand:VN 2 "register_operand" "w"))))]
5715  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5716  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5717  [(set_attr "neon_type" "neon_shift_1")
5718   (set_attr "length" "8")]
5719 )
5720
5721 ;; For the non-quad case.
5722 (define_insn "neon_vec_pack_trunc_<mode>"
5723  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5724        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5725  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5726  "vmovn.i<V_sz_elem>\t%P0, %q1"
5727  [(set_attr "neon_type" "neon_shift_1")]
5728 )
5729
5730 (define_expand "vec_pack_trunc_<mode>"
5731  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5732   (match_operand:VSHFT 1 "register_operand" "")
5733   (match_operand:VSHFT 2 "register_operand")]
5734  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5735 {
5736   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5737
5738   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5739   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5740   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5741   DONE;
5742 })
5743
5744 (define_insn "neon_vabd<mode>_2"
5745  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5746        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5747                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5748  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5749  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5750  [(set (attr "neon_type")
5751        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5752                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5753                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5754                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5755                      (const_string "neon_int_5")))]
5756 )
5757
5758 (define_insn "neon_vabd<mode>_3"
5759  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5760        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5761                              (match_operand:VDQ 2 "s_register_operand" "w")]
5762                  UNSPEC_VSUB)))]
5763  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5764  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5765  [(set (attr "neon_type")
5766        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5767                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5768                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5769                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5770                      (const_string "neon_int_5")))]
5771 )