gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006-2013 Free Software Foundation, Inc.
   3 ;; Written by CodeSourcery.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ;; Attribute used to permit string comparisons against <VQH_mnem> in
  23 ;; neon_type attribute definitions.
  24 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
  25
  26 (define_insn "*neon_mov<mode>"
  27   [(set (match_operand:VDX 0 "nonimmediate_operand"
  28           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
  29         (match_operand:VDX 1 "general_operand"
  30           " w,w, Dn,Uni, w, r, r, Usi,r"))]
  31   "TARGET_NEON
  32    && (register_operand (operands[0], <MODE>mode)
  33        || register_operand (operands[1], <MODE>mode))"
  34 {
  35   if (which_alternative == 2)
  36     {
  37       int width, is_valid;
  38       static char templ[40];
  39
  40       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  41         &operands[1], &width);
  42
  43       gcc_assert (is_valid != 0);
  44
  45       if (width == 0)
  46         return "vmov.f32\t%P0, %1  @ <mode>";
  47       else
  48         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
  49
  50       return templ;
  51     }
  52
  53   switch (which_alternative)
  54     {
  55     case 0: return "vmov\t%P0, %P1  @ <mode>";
  56     case 1: case 3: return output_move_neon (operands);
  57     case 2: gcc_unreachable ();
  58     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
  59     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
  60     default: return output_move_double (operands, true, NULL);
  61     }
  62 }
  63  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
  64   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu_reg,load2,store2")
  65   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
  66   (set_attr "length" "4,4,4,4,4,4,8,8,8")
  67   (set_attr "arm_pool_range"     "*,*,*,1020,*,*,*,1020,*")
  68   (set_attr "thumb2_pool_range"     "*,*,*,1018,*,*,*,1018,*")
  69   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
  70
  71 (define_insn "*neon_mov<mode>"
  72   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
  73           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
  74         (match_operand:VQXMOV 1 "general_operand"
  75           " w,w, Dn,Uni, w, r, r, Usi, r"))]
  76   "TARGET_NEON
  77    && (register_operand (operands[0], <MODE>mode)
  78        || register_operand (operands[1], <MODE>mode))"
  79 {
  80   if (which_alternative == 2)
  81     {
  82       int width, is_valid;
  83       static char templ[40];
  84
  85       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  86         &operands[1], &width);
  87
  88       gcc_assert (is_valid != 0);
  89
  90       if (width == 0)
  91         return "vmov.f32\t%q0, %1  @ <mode>";
  92       else
  93         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
  94
  95       return templ;
  96     }
  97
  98   switch (which_alternative)
  99     {
 100     case 0: return "vmov\t%q0, %q1  @ <mode>";
 101     case 1: case 3: return output_move_neon (operands);
 102     case 2: gcc_unreachable ();
 103     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 104     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 105     default: return output_move_quad (operands);
 106     }
 107 }
 108   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 109                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 110    (set_attr "type" "*,*,*,*,*,*,alu_reg,load4,store4")
 111    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 112    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 113    (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*")
 114    (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*")
 115    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 116
 117 (define_expand "movti"
 118   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 119         (match_operand:TI 1 "general_operand" ""))]
 120   "TARGET_NEON"
 121 {
 122   if (can_create_pseudo_p ())
 123     {
 124       if (!REG_P (operands[0]))
 125         operands[1] = force_reg (TImode, operands[1]);
 126     }
 127 })
 128
 129 (define_expand "mov<mode>"
 130   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 131         (match_operand:VSTRUCT 1 "general_operand" ""))]
 132   "TARGET_NEON"
 133 {
 134   if (can_create_pseudo_p ())
 135     {
 136       if (!REG_P (operands[0]))
 137         operands[1] = force_reg (<MODE>mode, operands[1]);
 138     }
 139 })
 140
 141 (define_insn "*neon_mov<mode>"
 142   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 143         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 144   "TARGET_NEON
 145    && (register_operand (operands[0], <MODE>mode)
 146        || register_operand (operands[1], <MODE>mode))"
 147 {
 148   switch (which_alternative)
 149     {
 150     case 0: return "#";
 151     case 1: case 2: return output_move_neon (operands);
 152     default: gcc_unreachable ();
 153     }
 154 }
 155   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 156    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 157
 158 (define_split
 159   [(set (match_operand:EI 0 "s_register_operand" "")
 160         (match_operand:EI 1 "s_register_operand" ""))]
 161   "TARGET_NEON && reload_completed"
 162   [(set (match_dup 0) (match_dup 1))
 163    (set (match_dup 2) (match_dup 3))]
 164 {
 165   int rdest = REGNO (operands[0]);
 166   int rsrc = REGNO (operands[1]);
 167   rtx dest[2], src[2];
 168
 169   dest[0] = gen_rtx_REG (TImode, rdest);
 170   src[0] = gen_rtx_REG (TImode, rsrc);
 171   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 172   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 173
 174   neon_disambiguate_copy (operands, dest, src, 2);
 175 })
 176
 177 (define_split
 178   [(set (match_operand:OI 0 "s_register_operand" "")
 179         (match_operand:OI 1 "s_register_operand" ""))]
 180   "TARGET_NEON && reload_completed"
 181   [(set (match_dup 0) (match_dup 1))
 182    (set (match_dup 2) (match_dup 3))]
 183 {
 184   int rdest = REGNO (operands[0]);
 185   int rsrc = REGNO (operands[1]);
 186   rtx dest[2], src[2];
 187
 188   dest[0] = gen_rtx_REG (TImode, rdest);
 189   src[0] = gen_rtx_REG (TImode, rsrc);
 190   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 191   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 192
 193   neon_disambiguate_copy (operands, dest, src, 2);
 194 })
 195
 196 (define_split
 197   [(set (match_operand:CI 0 "s_register_operand" "")
 198         (match_operand:CI 1 "s_register_operand" ""))]
 199   "TARGET_NEON && reload_completed"
 200   [(set (match_dup 0) (match_dup 1))
 201    (set (match_dup 2) (match_dup 3))
 202    (set (match_dup 4) (match_dup 5))]
 203 {
 204   int rdest = REGNO (operands[0]);
 205   int rsrc = REGNO (operands[1]);
 206   rtx dest[3], src[3];
 207
 208   dest[0] = gen_rtx_REG (TImode, rdest);
 209   src[0] = gen_rtx_REG (TImode, rsrc);
 210   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 211   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 212   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 213   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 214
 215   neon_disambiguate_copy (operands, dest, src, 3);
 216 })
 217
 218 (define_split
 219   [(set (match_operand:XI 0 "s_register_operand" "")
 220         (match_operand:XI 1 "s_register_operand" ""))]
 221   "TARGET_NEON && reload_completed"
 222   [(set (match_dup 0) (match_dup 1))
 223    (set (match_dup 2) (match_dup 3))
 224    (set (match_dup 4) (match_dup 5))
 225    (set (match_dup 6) (match_dup 7))]
 226 {
 227   int rdest = REGNO (operands[0]);
 228   int rsrc = REGNO (operands[1]);
 229   rtx dest[4], src[4];
 230
 231   dest[0] = gen_rtx_REG (TImode, rdest);
 232   src[0] = gen_rtx_REG (TImode, rsrc);
 233   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 234   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 235   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 236   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 237   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 238   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 239
 240   neon_disambiguate_copy (operands, dest, src, 4);
 241 })
 242
 243 (define_expand "movmisalign<mode>"
 244   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 245         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 246                      UNSPEC_MISALIGNED_ACCESS))]
 247   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 248 {
 249   /* This pattern is not permitted to fail during expansion: if both arguments
 250      are non-registers (e.g. memory := constant, which can be created by the
 251      auto-vectorizer), force operand 1 into a register.  */
 252   if (!s_register_operand (operands[0], <MODE>mode)
 253       && !s_register_operand (operands[1], <MODE>mode))
 254     operands[1] = force_reg (<MODE>mode, operands[1]);
 255 })
 256
 257 (define_insn "*movmisalign<mode>_neon_store"
 258   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 259         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 260                     UNSPEC_MISALIGNED_ACCESS))]
 261   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 262   "vst1.<V_sz_elem>\t{%P1}, %A0"
 263   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 264
 265 (define_insn "*movmisalign<mode>_neon_load"
 266   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 267         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 268                     UNSPEC_MISALIGNED_ACCESS))]
 269   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 270   "vld1.<V_sz_elem>\t{%P0}, %A1"
 271   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 272
 273 (define_insn "*movmisalign<mode>_neon_store"
 274   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 275         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 276                     UNSPEC_MISALIGNED_ACCESS))]
 277   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 278   "vst1.<V_sz_elem>\t{%q1}, %A0"
 279   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 280
 281 (define_insn "*movmisalign<mode>_neon_load"
 282   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 283         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 284                     UNSPEC_MISALIGNED_ACCESS))]
 285   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 286   "vld1.<V_sz_elem>\t{%q0}, %A1"
 287   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 288
 289 (define_insn "vec_set<mode>_internal"
 290   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 291         (vec_merge:VD
 292           (vec_duplicate:VD
 293             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 294           (match_operand:VD 3 "s_register_operand" "0,0")
 295           (match_operand:SI 2 "immediate_operand" "i,i")))]
 296   "TARGET_NEON"
 297 {
 298   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 299   if (BYTES_BIG_ENDIAN)
 300     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 301   operands[2] = GEN_INT (elt);
 302
 303   if (which_alternative == 0)
 304     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 305   else
 306     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 307 }
 308   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")])
 309
 310 (define_insn "vec_set<mode>_internal"
 311   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 312         (vec_merge:VQ
 313           (vec_duplicate:VQ
 314             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 315           (match_operand:VQ 3 "s_register_operand" "0,0")
 316           (match_operand:SI 2 "immediate_operand" "i,i")))]
 317   "TARGET_NEON"
 318 {
 319   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 320   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 321   int elt = elem % half_elts;
 322   int hi = (elem / half_elts) * 2;
 323   int regno = REGNO (operands[0]);
 324
 325   if (BYTES_BIG_ENDIAN)
 326     elt = half_elts - 1 - elt;
 327
 328   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 329   operands[2] = GEN_INT (elt);
 330
 331   if (which_alternative == 0)
 332     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 333   else
 334     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 335 }
 336   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
 337 )
 338
 339 (define_insn "vec_setv2di_internal"
 340   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 341         (vec_merge:V2DI
 342           (vec_duplicate:V2DI
 343             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 344           (match_operand:V2DI 3 "s_register_operand" "0,0")
 345           (match_operand:SI 2 "immediate_operand" "i,i")))]
 346   "TARGET_NEON"
 347 {
 348   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 349   int regno = REGNO (operands[0]) + 2 * elem;
 350
 351   operands[0] = gen_rtx_REG (DImode, regno);
 352
 353   if (which_alternative == 0)
 354     return "vld1.64\t%P0, %A1";
 355   else
 356     return "vmov\t%P0, %Q1, %R1";
 357 }
 358   [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")]
 359 )
 360
 361 (define_expand "vec_set<mode>"
 362   [(match_operand:VDQ 0 "s_register_operand" "")
 363    (match_operand:<V_elem> 1 "s_register_operand" "")
 364    (match_operand:SI 2 "immediate_operand" "")]
 365   "TARGET_NEON"
 366 {
 367   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 368   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 369                                          GEN_INT (elem), operands[0]));
 370   DONE;
 371 })
 372
 373 (define_insn "vec_extract<mode>"
 374   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 375         (vec_select:<V_elem>
 376           (match_operand:VD 1 "s_register_operand" "w,w")
 377           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 378   "TARGET_NEON"
 379 {
 380   if (BYTES_BIG_ENDIAN)
 381     {
 382       int elt = INTVAL (operands[2]);
 383       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 384       operands[2] = GEN_INT (elt);
 385     }
 386
 387   if (which_alternative == 0)
 388     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 389   else
 390     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 391 }
 392   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 393 )
 394
 395 (define_insn "vec_extract<mode>"
 396   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 397         (vec_select:<V_elem>
 398           (match_operand:VQ 1 "s_register_operand" "w,w")
 399           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 400   "TARGET_NEON"
 401 {
 402   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 403   int elt = INTVAL (operands[2]) % half_elts;
 404   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 405   int regno = REGNO (operands[1]);
 406
 407   if (BYTES_BIG_ENDIAN)
 408     elt = half_elts - 1 - elt;
 409
 410   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 411   operands[2] = GEN_INT (elt);
 412
 413   if (which_alternative == 0)
 414     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 415   else
 416     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 417 }
 418   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 419 )
 420
 421 (define_insn "vec_extractv2di"
 422   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 423         (vec_select:DI
 424           (match_operand:V2DI 1 "s_register_operand" "w,w")
 425           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 426   "TARGET_NEON"
 427 {
 428   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 429
 430   operands[1] = gen_rtx_REG (DImode, regno);
 431
 432   if (which_alternative == 0)
 433     return "vst1.64\t{%P1}, %A0  @ v2di";
 434   else
 435     return "vmov\t%Q0, %R0, %P1  @ v2di";
 436 }
 437   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")]
 438 )
 439
 440 (define_expand "vec_init<mode>"
 441   [(match_operand:VDQ 0 "s_register_operand" "")
 442    (match_operand 1 "" "")]
 443   "TARGET_NEON"
 444 {
 445   neon_expand_vector_init (operands[0], operands[1]);
 446   DONE;
 447 })
 448
 449 ;; Doubleword and quadword arithmetic.
 450
 451 ;; NOTE: some other instructions also support 64-bit integer
 452 ;; element size, which we could potentially use for "long long" operations.
 453
 454 (define_insn "*add<mode>3_neon"
 455   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 456         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 457                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 458   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 459   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 460   [(set (attr "neon_type")
 461       (if_then_else (match_test "<Is_float_mode>")
 462                     (if_then_else (match_test "<Is_d_reg>")
 463                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 464                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 465                     (const_string "neon_int_1")))]
 466 )
 467
 468 (define_insn "adddi3_neon"
 469   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 470         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 471                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 472    (clobber (reg:CC CC_REGNUM))]
 473   "TARGET_NEON"
 474 {
 475   switch (which_alternative)
 476     {
 477     case 0: /* fall through */
 478     case 3: return "vadd.i64\t%P0, %P1, %P2";
 479     case 1: return "#";
 480     case 2: return "#";
 481     case 4: return "#";
 482     case 5: return "#";
 483     case 6: return "#";
 484     default: gcc_unreachable ();
 485     }
 486 }
 487   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 488    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 489    (set_attr "length" "*,8,8,*,8,8,8")
 490    (set_attr "arch" "neon_for_64bits,*,*,avoid_neon_for_64bits,*,*,*")]
 491 )
 492
 493 (define_insn "*sub<mode>3_neon"
 494   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 495         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 496                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 497   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 498   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 499   [(set (attr "neon_type")
 500       (if_then_else (match_test "<Is_float_mode>")
 501                     (if_then_else (match_test "<Is_d_reg>")
 502                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 503                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 504                     (const_string "neon_int_2")))]
 505 )
 506
 507 (define_insn "subdi3_neon"
 508   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 509         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 510                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 511    (clobber (reg:CC CC_REGNUM))]
 512   "TARGET_NEON"
 513 {
 514   switch (which_alternative)
 515     {
 516     case 0: /* fall through */
 517     case 4: return "vsub.i64\t%P0, %P1, %P2";
 518     case 1: /* fall through */
 519     case 2: /* fall through */
 520     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 521     default: gcc_unreachable ();
 522     }
 523 }
 524   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 525    (set_attr "conds" "*,clob,clob,clob,*")
 526    (set_attr "length" "*,8,8,8,*")
 527    (set_attr "arch" "neon_for_64bits,*,*,*,avoid_neon_for_64bits")]
 528 )
 529
 530 (define_insn "*mul<mode>3_neon"
 531   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 532         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 533                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 534   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 535   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 536   [(set (attr "neon_type")
 537       (if_then_else (match_test "<Is_float_mode>")
 538                     (if_then_else (match_test "<Is_d_reg>")
 539                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 540                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 541                     (if_then_else (match_test "<Is_d_reg>")
 542                                   (if_then_else
 543                                     (match_test "<Scalar_mul_8_16>")
 544                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 545                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 546                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 547                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 548                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 549 )
 550
 551 (define_insn "mul<mode>3add<mode>_neon"
 552   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 553         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 554                             (match_operand:VDQ 3 "s_register_operand" "w"))
 555                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 556   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 557   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 558   [(set (attr "neon_type")
 559       (if_then_else (match_test "<Is_float_mode>")
 560                     (if_then_else (match_test "<Is_d_reg>")
 561                                   (const_string "neon_fp_vmla_ddd")
 562                                   (const_string "neon_fp_vmla_qqq"))
 563                     (if_then_else (match_test "<Is_d_reg>")
 564                                   (if_then_else
 565                                     (match_test "<Scalar_mul_8_16>")
 566                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 567                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 568                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 569                                     (const_string "neon_mla_qqq_8_16")
 570                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 571 )
 572
 573 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 574   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 575         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 576                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 577                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 578   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 579   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 580   [(set (attr "neon_type")
 581       (if_then_else (match_test "<Is_float_mode>")
 582                     (if_then_else (match_test "<Is_d_reg>")
 583                                   (const_string "neon_fp_vmla_ddd")
 584                                   (const_string "neon_fp_vmla_qqq"))
 585                     (if_then_else (match_test "<Is_d_reg>")
 586                                   (if_then_else
 587                                     (match_test "<Scalar_mul_8_16>")
 588                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 589                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 590                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 591                                     (const_string "neon_mla_qqq_8_16")
 592                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 593 )
 594
 595 ;; Fused multiply-accumulate
 596 ;; We define each insn twice here:
 597 ;;    1: with flag_unsafe_math_optimizations for the widening multiply phase
 598 ;;       to be able to use when converting to FMA.
 599 ;;    2: without flag_unsafe_math_optimizations for the intrinsics to use.
 600 (define_insn "fma<VCVTF:mode>4"
 601   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 602         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 603                  (match_operand:VCVTF 2 "register_operand" "w")
 604                  (match_operand:VCVTF 3 "register_operand" "0")))]
 605   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 606   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 607   [(set (attr "neon_type")
 608         (if_then_else (match_test "<Is_d_reg>")
 609                       (const_string "neon_fp_vmla_ddd")
 610                       (const_string "neon_fp_vmla_qqq")))]
 611 )
 612
 613 (define_insn "fma<VCVTF:mode>4_intrinsic"
 614   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 615         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 616                  (match_operand:VCVTF 2 "register_operand" "w")
 617                  (match_operand:VCVTF 3 "register_operand" "0")))]
 618   "TARGET_NEON && TARGET_FMA"
 619   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 620   [(set (attr "neon_type")
 621         (if_then_else (match_test "<Is_d_reg>")
 622                       (const_string "neon_fp_vmla_ddd")
 623                       (const_string "neon_fp_vmla_qqq")))]
 624 )
 625
 626 (define_insn "*fmsub<VCVTF:mode>4"
 627   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 628         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 629                    (match_operand:VCVTF 2 "register_operand" "w")
 630                    (match_operand:VCVTF 3 "register_operand" "0")))]
 631   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 632   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 633   [(set (attr "neon_type")
 634         (if_then_else (match_test "<Is_d_reg>")
 635                       (const_string "neon_fp_vmla_ddd")
 636                       (const_string "neon_fp_vmla_qqq")))]
 637 )
 638
 639 (define_insn "fmsub<VCVTF:mode>4_intrinsic"
 640   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 641         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 642                    (match_operand:VCVTF 2 "register_operand" "w")
 643                    (match_operand:VCVTF 3 "register_operand" "0")))]
 644   "TARGET_NEON && TARGET_FMA"
 645   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 646   [(set (attr "neon_type")
 647         (if_then_else (match_test "<Is_d_reg>")
 648                       (const_string "neon_fp_vmla_ddd")
 649                       (const_string "neon_fp_vmla_qqq")))]
 650 )
 651
 652 (define_insn "neon_vrint<NEON_VRINT:nvrint_variant><VCVTF:mode>"
 653   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
 654         (unspec:VCVTF [(match_operand:VCVTF 1
 655                          "s_register_operand" "w")]
 656                 NEON_VRINT))]
 657   "TARGET_NEON && TARGET_FPU_ARMV8"
 658   "vrint<nvrint_variant>%?.f32\\t%<V_reg>0, %<V_reg>1"
 659   [(set (attr "neon_type")
 660         (if_then_else (match_test "<Is_d_reg>")
 661                 (const_string "neon_fp_vadd_ddd_vabs_dd")
 662                 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
 663 )
 664
 665 (define_insn "ior<mode>3"
 666   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 667         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 668                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 669   "TARGET_NEON"
 670 {
 671   switch (which_alternative)
 672     {
 673     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 674     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 675                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 676     default: gcc_unreachable ();
 677     }
 678 }
 679   [(set_attr "neon_type" "neon_int_1")]
 680 )
 681
 682 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 683 ;; vorr. We support the pseudo-instruction vand instead, because that
 684 ;; corresponds to the canonical form the middle-end expects to use for
 685 ;; immediate bitwise-ANDs.
 686
 687 (define_insn "and<mode>3"
 688   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 689         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 690                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 691   "TARGET_NEON"
 692 {
 693   switch (which_alternative)
 694     {
 695     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 696     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 697                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 698     default: gcc_unreachable ();
 699     }
 700 }
 701   [(set_attr "neon_type" "neon_int_1")]
 702 )
 703
 704 (define_insn "orn<mode>3_neon"
 705   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 706         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 707                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 708   "TARGET_NEON"
 709   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 710   [(set_attr "neon_type" "neon_int_1")]
 711 )
 712
 713 ;; TODO: investigate whether we should disable
 714 ;; this and bicdi3_neon for the A8 in line with the other
 715 ;; changes above.
 716 (define_insn_and_split "orndi3_neon"
 717   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 718         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 719                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 720   "TARGET_NEON"
 721   "@
 722    vorn\t%P0, %P1, %P2
 723    #
 724    #
 725    #"
 726   "reload_completed &&
 727    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 728   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 729    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 730   "
 731   {
 732     if (TARGET_THUMB2)
 733       {
 734         operands[3] = gen_highpart (SImode, operands[0]);
 735         operands[0] = gen_lowpart (SImode, operands[0]);
 736         operands[4] = gen_highpart (SImode, operands[2]);
 737         operands[2] = gen_lowpart (SImode, operands[2]);
 738         operands[5] = gen_highpart (SImode, operands[1]);
 739         operands[1] = gen_lowpart (SImode, operands[1]);
 740       }
 741     else
 742       {
 743         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 744         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 745         DONE;
 746       }
 747   }"
 748   [(set_attr "neon_type" "neon_int_1,*,*,*")
 749    (set_attr "length" "*,16,8,8")
 750    (set_attr "arch" "any,a,t2,t2")]
 751 )
 752
 753 (define_insn "bic<mode>3_neon"
 754   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 755         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 756                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 757   "TARGET_NEON"
 758   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 759   [(set_attr "neon_type" "neon_int_1")]
 760 )
 761
 762 ;; Compare to *anddi_notdi_di.
 763 (define_insn "bicdi3_neon"
 764   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 765         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 766                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 767   "TARGET_NEON"
 768   "@
 769    vbic\t%P0, %P1, %P2
 770    #
 771    #"
 772   [(set_attr "neon_type" "neon_int_1,*,*")
 773    (set_attr "length" "*,8,8")]
 774 )
 775
 776 (define_insn "xor<mode>3"
 777   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 778         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 779                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 780   "TARGET_NEON"
 781   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 782   [(set_attr "neon_type" "neon_int_1")]
 783 )
 784
 785 (define_insn "one_cmpl<mode>2"
 786   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 787         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 788   "TARGET_NEON"
 789   "vmvn\t%<V_reg>0, %<V_reg>1"
 790   [(set_attr "neon_type" "neon_int_1")]
 791 )
 792
 793 (define_insn "abs<mode>2"
 794   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 795         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 796   "TARGET_NEON"
 797   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 798   [(set (attr "neon_type")
 799       (if_then_else (match_test "<Is_float_mode>")
 800                     (if_then_else (match_test "<Is_d_reg>")
 801                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 802                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 803                     (const_string "neon_int_3")))]
 804 )
 805
 806 (define_insn "neg<mode>2"
 807   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 808         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 809   "TARGET_NEON"
 810   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 811   [(set (attr "neon_type")
 812       (if_then_else (match_test "<Is_float_mode>")
 813                     (if_then_else (match_test "<Is_d_reg>")
 814                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 815                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 816                     (const_string "neon_int_3")))]
 817 )
 818
 819 (define_insn "negdi2_neon"
 820   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 821         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 822    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 823    (clobber (reg:CC CC_REGNUM))]
 824   "TARGET_NEON"
 825   "#"
 826   [(set_attr "length" "8")]
 827 )
 828
 829 ; Split negdi2_neon for vfp registers
 830 (define_split
 831   [(set (match_operand:DI 0 "s_register_operand" "")
 832         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 833    (clobber (match_scratch:DI 2 ""))
 834    (clobber (reg:CC CC_REGNUM))]
 835   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 836   [(set (match_dup 2) (const_int 0))
 837    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 838               (clobber (reg:CC CC_REGNUM))])]
 839   {
 840     if (!REG_P (operands[2]))
 841       operands[2] = operands[0];
 842   }
 843 )
 844
 845 ; Split negdi2_neon for core registers
 846 (define_split
 847   [(set (match_operand:DI 0 "s_register_operand" "")
 848         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 849    (clobber (match_scratch:DI 2 ""))
 850    (clobber (reg:CC CC_REGNUM))]
 851   "TARGET_32BIT && reload_completed
 852    && arm_general_register_operand (operands[0], DImode)"
 853   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 854               (clobber (reg:CC CC_REGNUM))])]
 855   ""
 856 )
 857
 858 (define_insn "*umin<mode>3_neon"
 859   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 860         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 861                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 862   "TARGET_NEON"
 863   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 864   [(set_attr "neon_type" "neon_int_5")]
 865 )
 866
 867 (define_insn "*umax<mode>3_neon"
 868   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 869         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 870                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 871   "TARGET_NEON"
 872   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 873   [(set_attr "neon_type" "neon_int_5")]
 874 )
 875
 876 (define_insn "*smin<mode>3_neon"
 877   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 878         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 879                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 880   "TARGET_NEON"
 881   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 882   [(set (attr "neon_type")
 883       (if_then_else (match_test "<Is_float_mode>")
 884                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 885                     (const_string "neon_int_5")))]
 886 )
 887
 888 (define_insn "*smax<mode>3_neon"
 889   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 890         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 891                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 892   "TARGET_NEON"
 893   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 894   [(set (attr "neon_type")
 895       (if_then_else (match_test "<Is_float_mode>")
 896                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 897                     (const_string "neon_int_5")))]
 898 )
 899
 900 ; TODO: V2DI shifts are current disabled because there are bugs in the
 901 ; generic vectorizer code.  It ends up creating a V2DI constructor with
 902 ; SImode elements.
 903
 904 (define_insn "vashl<mode>3"
 905   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
 906         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
 907                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
 908   "TARGET_NEON"
 909   {
 910     switch (which_alternative)
 911       {
 912         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 913         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
 914                                                     <MODE>mode,
 915                                                     VALID_NEON_QREG_MODE (<MODE>mode),
 916                                                     true);
 917         default: gcc_unreachable ();
 918       }
 919   }
 920   [(set (attr "neon_type")
 921       (if_then_else (match_test "<Is_d_reg>")
 922                     (const_string "neon_vshl_ddd")
 923                     (const_string "neon_shift_3")))]
 924 )
 925
 926 (define_insn "vashr<mode>3_imm"
 927   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 928         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 929                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 930   "TARGET_NEON"
 931   {
 932     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 933                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 934                                         false);
 935   }
 936   [(set (attr "neon_type")
 937       (if_then_else (match_test "<Is_d_reg>")
 938                     (const_string "neon_vshl_ddd")
 939                     (const_string "neon_shift_3")))]
 940 )
 941
 942 (define_insn "vlshr<mode>3_imm"
 943   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 944         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 945                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 946   "TARGET_NEON"
 947   {
 948     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 949                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 950                                         false);
 951   }
 952   [(set (attr "neon_type")
 953         (if_then_else (match_test "<Is_d_reg>")
 954                       (const_string "neon_vshl_ddd")
 955                       (const_string "neon_shift_3")))]
 956 )
 957
 958 ; Used for implementing logical shift-right, which is a left-shift by a negative
 959 ; amount, with signed operands. This is essentially the same as ashl<mode>3
 960 ; above, but using an unspec in case GCC tries anything tricky with negative
 961 ; shift amounts.
 962
 963 (define_insn "ashl<mode>3_signed"
 964   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 965         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 966                       (match_operand:VDQI 2 "s_register_operand" "w")]
 967                      UNSPEC_ASHIFT_SIGNED))]
 968   "TARGET_NEON"
 969   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 970   [(set (attr "neon_type")
 971       (if_then_else (match_test "<Is_d_reg>")
 972                     (const_string "neon_vshl_ddd")
 973                     (const_string "neon_shift_3")))]
 974 )
 975
 976 ; Used for implementing logical shift-right, which is a left-shift by a negative
 977 ; amount, with unsigned operands.
 978
 979 (define_insn "ashl<mode>3_unsigned"
 980   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 981         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 982                       (match_operand:VDQI 2 "s_register_operand" "w")]
 983                      UNSPEC_ASHIFT_UNSIGNED))]
 984   "TARGET_NEON"
 985   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 986   [(set (attr "neon_type")
 987       (if_then_else (match_test "<Is_d_reg>")
 988                     (const_string "neon_vshl_ddd")
 989                     (const_string "neon_shift_3")))]
 990 )
 991
 992 (define_expand "vashr<mode>3"
 993   [(set (match_operand:VDQIW 0 "s_register_operand" "")
 994         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
 995                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
 996   "TARGET_NEON"
 997 {
 998   if (s_register_operand (operands[2], <MODE>mode))
 999     {
1000       rtx neg = gen_reg_rtx (<MODE>mode);
1001       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1002       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1003     }
1004   else
1005     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1006   DONE;
1007 })
1008
1009 (define_expand "vlshr<mode>3"
1010   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1011         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1012                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1013   "TARGET_NEON"
1014 {
1015   if (s_register_operand (operands[2], <MODE>mode))
1016     {
1017       rtx neg = gen_reg_rtx (<MODE>mode);
1018       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1019       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1020     }
1021   else
1022     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1023   DONE;
1024 })
1025
1026 ;; 64-bit shifts
1027
1028 ;; This pattern loads a 32-bit shift count into a 64-bit NEON register,
1029 ;; leaving the upper half uninitalized.  This is OK since the shift
1030 ;; instruction only looks at the low 8 bits anyway.  To avoid confusing
1031 ;; data flow analysis however, we pretend the full register is set
1032 ;; using an unspec.
1033 (define_insn "neon_load_count"
1034   [(set (match_operand:DI 0 "s_register_operand" "=w,w")
1035         (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")]
1036                    UNSPEC_LOAD_COUNT))]
1037   "TARGET_NEON"
1038   "@
1039    vld1.32\t{%P0[0]}, %A1
1040    vmov.32\t%P0[0], %1"
1041   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
1042 )
1043
1044 (define_insn "ashldi3_neon_noclobber"
1045   [(set (match_operand:DI 0 "s_register_operand"            "=w,w")
1046         (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w")
1047                    (match_operand:DI 2 "reg_or_int_operand" " i,w")))]
1048   "TARGET_NEON && reload_completed
1049    && (!CONST_INT_P (operands[2])
1050        || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))"
1051   "@
1052    vshl.u64\t%P0, %P1, %2
1053    vshl.u64\t%P0, %P1, %P2"
1054   [(set_attr "neon_type" "neon_vshl_ddd,neon_vshl_ddd")]
1055 )
1056
1057 (define_insn_and_split "ashldi3_neon"
1058   [(set (match_operand:DI 0 "s_register_operand"            "= w, w,?&r,?r, ?w,w")
1059         (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w")
1060                    (match_operand:SI 2 "general_operand"    "rUm, i,  r, i,rUm,i")))
1061    (clobber (match_scratch:SI 3                             "= X, X,?&r, X,  X,X"))
1062    (clobber (match_scratch:SI 4                             "= X, X,?&r, X,  X,X"))
1063    (clobber (match_scratch:DI 5                             "=&w, X,  X, X, &w,X"))
1064    (clobber (reg:CC_C CC_REGNUM))]
1065   "TARGET_NEON"
1066   "#"
1067   "TARGET_NEON && reload_completed"
1068   [(const_int 0)]
1069   "
1070   {
1071     if (IS_VFP_REGNUM (REGNO (operands[0])))
1072       {
1073         if (CONST_INT_P (operands[2]))
1074           {
1075             if (INTVAL (operands[2]) < 1)
1076               {
1077                 emit_insn (gen_movdi (operands[0], operands[1]));
1078                 DONE;
1079               }
1080             else if (INTVAL (operands[2]) > 63)
1081               operands[2] = gen_rtx_CONST_INT (VOIDmode, 63);
1082           }
1083         else
1084           {
1085             emit_insn (gen_neon_load_count (operands[5], operands[2]));
1086             operands[2] = operands[5];
1087           }
1088
1089         /* Ditch the unnecessary clobbers.  */
1090         emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1],
1091                                                operands[2]));
1092       }
1093     else
1094       {
1095         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1096           /* This clobbers CC.  */
1097           emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
1098         else
1099           arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
1100                                          operands[2], operands[3], operands[4]);
1101       }
1102     DONE;
1103   }"
1104   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1105    (set_attr "opt" "*,*,speed,speed,*,*")]
1106 )
1107
1108 ; The shift amount needs to be negated for right-shifts
1109 (define_insn "signed_shift_di3_neon"
1110   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1111         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1112                     (match_operand:DI 2 "s_register_operand" " w")]
1113                    UNSPEC_ASHIFT_SIGNED))]
1114   "TARGET_NEON && reload_completed"
1115   "vshl.s64\t%P0, %P1, %P2"
1116   [(set_attr "neon_type" "neon_vshl_ddd")]
1117 )
1118
1119 ; The shift amount needs to be negated for right-shifts
1120 (define_insn "unsigned_shift_di3_neon"
1121   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1122         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1123                     (match_operand:DI 2 "s_register_operand" " w")]
1124                    UNSPEC_ASHIFT_UNSIGNED))]
1125   "TARGET_NEON && reload_completed"
1126   "vshl.u64\t%P0, %P1, %P2"
1127   [(set_attr "neon_type" "neon_vshl_ddd")]
1128 )
1129
1130 (define_insn "ashrdi3_neon_imm_noclobber"
1131   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1132         (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1133                      (match_operand:DI 2 "const_int_operand"  " i")))]
1134   "TARGET_NEON && reload_completed
1135    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1136   "vshr.s64\t%P0, %P1, %2"
1137   [(set_attr "neon_type" "neon_vshl_ddd")]
1138 )
1139
1140 (define_insn "lshrdi3_neon_imm_noclobber"
1141   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1142         (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1143                      (match_operand:DI 2 "const_int_operand"  " i")))]
1144   "TARGET_NEON && reload_completed
1145    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1146   "vshr.u64\t%P0, %P1, %2"
1147   [(set_attr "neon_type" "neon_vshl_ddd")]
1148 )
1149
1150 ;; ashrdi3_neon
1151 ;; lshrdi3_neon
1152 (define_insn_and_split "<shift>di3_neon"
1153   [(set (match_operand:DI 0 "s_register_operand"             "= w, w,?&r,?r,?w,?w")
1154         (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w")
1155                     (match_operand:SI 2 "reg_or_int_operand" "  r, i,  r, i, r, i")))
1156    (clobber (match_scratch:SI 3                              "=2r, X, &r, X,2r, X"))
1157    (clobber (match_scratch:SI 4                              "= X, X, &r, X, X, X"))
1158    (clobber (match_scratch:DI 5                              "=&w, X,  X, X,&w, X"))
1159    (clobber (reg:CC CC_REGNUM))]
1160   "TARGET_NEON"
1161   "#"
1162   "TARGET_NEON && reload_completed"
1163   [(const_int 0)]
1164   "
1165   {
1166     if (IS_VFP_REGNUM (REGNO (operands[0])))
1167       {
1168         if (CONST_INT_P (operands[2]))
1169           {
1170             if (INTVAL (operands[2]) < 1)
1171               {
1172                 emit_insn (gen_movdi (operands[0], operands[1]));
1173                 DONE;
1174               }
1175             else if (INTVAL (operands[2]) > 64)
1176               operands[2] = gen_rtx_CONST_INT (VOIDmode, 64);
1177
1178             /* Ditch the unnecessary clobbers.  */
1179             emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0],
1180                                                           operands[1],
1181                                                           operands[2]));
1182           }
1183         else
1184           {
1185             /* We must use a negative left-shift.  */
1186             emit_insn (gen_negsi2 (operands[3], operands[2]));
1187             emit_insn (gen_neon_load_count (operands[5], operands[3]));
1188             emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1],
1189                                                        operands[5]));
1190           }
1191       }
1192     else
1193       {
1194         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1195           /* This clobbers CC.  */
1196           emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
1197         else
1198           /* This clobbers CC (ASHIFTRT by register only).  */
1199           arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
1200                                          operands[2], operands[3], operands[4]);
1201       }
1202
1203     DONE;
1204   }"
1205   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1206    (set_attr "opt" "*,*,speed,speed,*,*")]
1207 )
1208
1209 ;; Widening operations
1210
1211 (define_insn "widen_ssum<mode>3"
1212   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1213         (plus:<V_widen> (sign_extend:<V_widen>
1214                           (match_operand:VW 1 "s_register_operand" "%w"))
1215                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1216   "TARGET_NEON"
1217   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1218   [(set_attr "neon_type" "neon_int_3")]
1219 )
1220
1221 (define_insn "widen_usum<mode>3"
1222   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1223         (plus:<V_widen> (zero_extend:<V_widen>
1224                           (match_operand:VW 1 "s_register_operand" "%w"))
1225                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1226   "TARGET_NEON"
1227   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1228   [(set_attr "neon_type" "neon_int_3")]
1229 )
1230
1231 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1232 ;; shift-count granularity. That's good enough for the middle-end's current
1233 ;; needs.
1234
1235 ;; Note that it's not safe to perform such an operation in big-endian mode,
1236 ;; due to element-ordering issues.
1237
1238 (define_expand "vec_shr_<mode>"
1239   [(match_operand:VDQ 0 "s_register_operand" "")
1240    (match_operand:VDQ 1 "s_register_operand" "")
1241    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1242   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1243 {
1244   rtx zero_reg;
1245   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1246   const int width = GET_MODE_BITSIZE (<MODE>mode);
1247   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1248   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1249     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1250
1251   if (num_bits == width)
1252     {
1253       emit_move_insn (operands[0], operands[1]);
1254       DONE;
1255     }
1256
1257   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1258   operands[0] = gen_lowpart (bvecmode, operands[0]);
1259   operands[1] = gen_lowpart (bvecmode, operands[1]);
1260
1261   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1262                       GEN_INT (num_bits / BITS_PER_UNIT)));
1263   DONE;
1264 })
1265
1266 (define_expand "vec_shl_<mode>"
1267   [(match_operand:VDQ 0 "s_register_operand" "")
1268    (match_operand:VDQ 1 "s_register_operand" "")
1269    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1270   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1271 {
1272   rtx zero_reg;
1273   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1274   const int width = GET_MODE_BITSIZE (<MODE>mode);
1275   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1276   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1277     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1278
1279   if (num_bits == 0)
1280     {
1281       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1282       DONE;
1283     }
1284
1285   num_bits = width - num_bits;
1286
1287   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1288   operands[0] = gen_lowpart (bvecmode, operands[0]);
1289   operands[1] = gen_lowpart (bvecmode, operands[1]);
1290
1291   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1292                       GEN_INT (num_bits / BITS_PER_UNIT)));
1293   DONE;
1294 })
1295
1296 ;; Helpers for quad-word reduction operations
1297
1298 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1299 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1300 ; N/2-element vector.
1301
1302 (define_insn "quad_halves_<code>v4si"
1303   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1304         (vqh_ops:V2SI
1305           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1306                            (parallel [(const_int 0) (const_int 1)]))
1307           (vec_select:V2SI (match_dup 1)
1308                            (parallel [(const_int 2) (const_int 3)]))))]
1309   "TARGET_NEON"
1310   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1311   [(set_attr "vqh_mnem" "<VQH_mnem>")
1312    (set (attr "neon_type")
1313       (if_then_else (eq_attr "vqh_mnem" "vadd")
1314                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1315 )
1316
1317 (define_insn "quad_halves_<code>v4sf"
1318   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1319         (vqhs_ops:V2SF
1320           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1321                            (parallel [(const_int 0) (const_int 1)]))
1322           (vec_select:V2SF (match_dup 1)
1323                            (parallel [(const_int 2) (const_int 3)]))))]
1324   "TARGET_NEON && flag_unsafe_math_optimizations"
1325   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1326   [(set_attr "vqh_mnem" "<VQH_mnem>")
1327    (set (attr "neon_type")
1328       (if_then_else (eq_attr "vqh_mnem" "vadd")
1329                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1330 )
1331
1332 (define_insn "quad_halves_<code>v8hi"
1333   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1334         (vqh_ops:V4HI
1335           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1336                            (parallel [(const_int 0) (const_int 1)
1337                                       (const_int 2) (const_int 3)]))
1338           (vec_select:V4HI (match_dup 1)
1339                            (parallel [(const_int 4) (const_int 5)
1340                                       (const_int 6) (const_int 7)]))))]
1341   "TARGET_NEON"
1342   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1343   [(set_attr "vqh_mnem" "<VQH_mnem>")
1344    (set (attr "neon_type")
1345       (if_then_else (eq_attr "vqh_mnem" "vadd")
1346                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1347 )
1348
1349 (define_insn "quad_halves_<code>v16qi"
1350   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1351         (vqh_ops:V8QI
1352           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1353                            (parallel [(const_int 0) (const_int 1)
1354                                       (const_int 2) (const_int 3)
1355                                       (const_int 4) (const_int 5)
1356                                       (const_int 6) (const_int 7)]))
1357           (vec_select:V8QI (match_dup 1)
1358                            (parallel [(const_int 8) (const_int 9)
1359                                       (const_int 10) (const_int 11)
1360                                       (const_int 12) (const_int 13)
1361                                       (const_int 14) (const_int 15)]))))]
1362   "TARGET_NEON"
1363   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1364   [(set_attr "vqh_mnem" "<VQH_mnem>")
1365    (set (attr "neon_type")
1366       (if_then_else (eq_attr "vqh_mnem" "vadd")
1367                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1368 )
1369
1370 (define_expand "move_hi_quad_<mode>"
1371  [(match_operand:ANY128 0 "s_register_operand" "")
1372   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1373  "TARGET_NEON"
1374 {
1375   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1376                                        GET_MODE_SIZE (<V_HALF>mode)),
1377                   operands[1]);
1378   DONE;
1379 })
1380
1381 (define_expand "move_lo_quad_<mode>"
1382  [(match_operand:ANY128 0 "s_register_operand" "")
1383   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1384  "TARGET_NEON"
1385 {
1386   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1387                                        <MODE>mode, 0),
1388                   operands[1]);
1389   DONE;
1390 })
1391
1392 ;; Reduction operations
1393
1394 (define_expand "reduc_splus_<mode>"
1395   [(match_operand:VD 0 "s_register_operand" "")
1396    (match_operand:VD 1 "s_register_operand" "")]
1397   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1398 {
1399   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1400                         &gen_neon_vpadd_internal<mode>);
1401   DONE;
1402 })
1403
1404 (define_expand "reduc_splus_<mode>"
1405   [(match_operand:VQ 0 "s_register_operand" "")
1406    (match_operand:VQ 1 "s_register_operand" "")]
1407   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1408    && !BYTES_BIG_ENDIAN"
1409 {
1410   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1411   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1412
1413   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1414   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1415   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1416
1417   DONE;
1418 })
1419
1420 (define_insn "reduc_splus_v2di"
1421   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1422         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1423                      UNSPEC_VPADD))]
1424   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1425   "vadd.i64\t%e0, %e1, %f1"
1426   [(set_attr "neon_type" "neon_int_1")]
1427 )
1428
1429 ;; NEON does not distinguish between signed and unsigned addition except on
1430 ;; widening operations.
1431 (define_expand "reduc_uplus_<mode>"
1432   [(match_operand:VDQI 0 "s_register_operand" "")
1433    (match_operand:VDQI 1 "s_register_operand" "")]
1434   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1435 {
1436   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1437   DONE;
1438 })
1439
1440 (define_expand "reduc_smin_<mode>"
1441   [(match_operand:VD 0 "s_register_operand" "")
1442    (match_operand:VD 1 "s_register_operand" "")]
1443   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1444 {
1445   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1446                         &gen_neon_vpsmin<mode>);
1447   DONE;
1448 })
1449
1450 (define_expand "reduc_smin_<mode>"
1451   [(match_operand:VQ 0 "s_register_operand" "")
1452    (match_operand:VQ 1 "s_register_operand" "")]
1453   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1454    && !BYTES_BIG_ENDIAN"
1455 {
1456   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1457   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1458
1459   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1460   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1461   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1462
1463   DONE;
1464 })
1465
1466 (define_expand "reduc_smax_<mode>"
1467   [(match_operand:VD 0 "s_register_operand" "")
1468    (match_operand:VD 1 "s_register_operand" "")]
1469   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1470 {
1471   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1472                         &gen_neon_vpsmax<mode>);
1473   DONE;
1474 })
1475
1476 (define_expand "reduc_smax_<mode>"
1477   [(match_operand:VQ 0 "s_register_operand" "")
1478    (match_operand:VQ 1 "s_register_operand" "")]
1479   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1480    && !BYTES_BIG_ENDIAN"
1481 {
1482   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1483   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1484
1485   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1486   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1487   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1488
1489   DONE;
1490 })
1491
1492 (define_expand "reduc_umin_<mode>"
1493   [(match_operand:VDI 0 "s_register_operand" "")
1494    (match_operand:VDI 1 "s_register_operand" "")]
1495   "TARGET_NEON"
1496 {
1497   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1498                         &gen_neon_vpumin<mode>);
1499   DONE;
1500 })
1501
1502 (define_expand "reduc_umin_<mode>"
1503   [(match_operand:VQI 0 "s_register_operand" "")
1504    (match_operand:VQI 1 "s_register_operand" "")]
1505   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1506 {
1507   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1508   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1509
1510   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1511   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1512   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1513
1514   DONE;
1515 })
1516
1517 (define_expand "reduc_umax_<mode>"
1518   [(match_operand:VDI 0 "s_register_operand" "")
1519    (match_operand:VDI 1 "s_register_operand" "")]
1520   "TARGET_NEON"
1521 {
1522   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1523                         &gen_neon_vpumax<mode>);
1524   DONE;
1525 })
1526
1527 (define_expand "reduc_umax_<mode>"
1528   [(match_operand:VQI 0 "s_register_operand" "")
1529    (match_operand:VQI 1 "s_register_operand" "")]
1530   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1531 {
1532   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1533   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1534
1535   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1536   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1537   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1538
1539   DONE;
1540 })
1541
1542 (define_insn "neon_vpadd_internal<mode>"
1543   [(set (match_operand:VD 0 "s_register_operand" "=w")
1544         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1545                     (match_operand:VD 2 "s_register_operand" "w")]
1546                    UNSPEC_VPADD))]
1547   "TARGET_NEON"
1548   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1549   ;; Assume this schedules like vadd.
1550   [(set (attr "neon_type")
1551       (if_then_else (match_test "<Is_float_mode>")
1552                     (if_then_else (match_test "<Is_d_reg>")
1553                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1554                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1555                     (const_string "neon_int_1")))]
1556 )
1557
1558 (define_insn "neon_vpsmin<mode>"
1559   [(set (match_operand:VD 0 "s_register_operand" "=w")
1560         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1561                     (match_operand:VD 2 "s_register_operand" "w")]
1562                    UNSPEC_VPSMIN))]
1563   "TARGET_NEON"
1564   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1565   ;; Assume this schedules like vmin.
1566   [(set (attr "neon_type")
1567       (if_then_else (match_test "<Is_float_mode>")
1568                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1569                     (const_string "neon_int_5")))]
1570 )
1571
1572 (define_insn "neon_vpsmax<mode>"
1573   [(set (match_operand:VD 0 "s_register_operand" "=w")
1574         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1575                     (match_operand:VD 2 "s_register_operand" "w")]
1576                    UNSPEC_VPSMAX))]
1577   "TARGET_NEON"
1578   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1579   ;; Assume this schedules like vmax.
1580   [(set (attr "neon_type")
1581       (if_then_else (match_test "<Is_float_mode>")
1582                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1583                     (const_string "neon_int_5")))]
1584 )
1585
1586 (define_insn "neon_vpumin<mode>"
1587   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1588         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1589                      (match_operand:VDI 2 "s_register_operand" "w")]
1590                    UNSPEC_VPUMIN))]
1591   "TARGET_NEON"
1592   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1593   ;; Assume this schedules like umin.
1594   [(set_attr "neon_type" "neon_int_5")]
1595 )
1596
1597 (define_insn "neon_vpumax<mode>"
1598   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1599         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1600                      (match_operand:VDI 2 "s_register_operand" "w")]
1601                    UNSPEC_VPUMAX))]
1602   "TARGET_NEON"
1603   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1604   ;; Assume this schedules like umax.
1605   [(set_attr "neon_type" "neon_int_5")]
1606 )
1607
1608 ;; Saturating arithmetic
1609
1610 ; NOTE: Neon supports many more saturating variants of instructions than the
1611 ; following, but these are all GCC currently understands.
1612 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1613 ; yet either, although these patterns may be used by intrinsics when they're
1614 ; added.
1615
1616 (define_insn "*ss_add<mode>_neon"
1617   [(set (match_operand:VD 0 "s_register_operand" "=w")
1618        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1619                    (match_operand:VD 2 "s_register_operand" "w")))]
1620   "TARGET_NEON"
1621   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1622   [(set_attr "neon_type" "neon_int_4")]
1623 )
1624
1625 (define_insn "*us_add<mode>_neon"
1626   [(set (match_operand:VD 0 "s_register_operand" "=w")
1627        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1628                    (match_operand:VD 2 "s_register_operand" "w")))]
1629   "TARGET_NEON"
1630   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1631   [(set_attr "neon_type" "neon_int_4")]
1632 )
1633
1634 (define_insn "*ss_sub<mode>_neon"
1635   [(set (match_operand:VD 0 "s_register_operand" "=w")
1636        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1637                     (match_operand:VD 2 "s_register_operand" "w")))]
1638   "TARGET_NEON"
1639   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1640   [(set_attr "neon_type" "neon_int_5")]
1641 )
1642
1643 (define_insn "*us_sub<mode>_neon"
1644   [(set (match_operand:VD 0 "s_register_operand" "=w")
1645        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1646                     (match_operand:VD 2 "s_register_operand" "w")))]
1647   "TARGET_NEON"
1648   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1649   [(set_attr "neon_type" "neon_int_5")]
1650 )
1651
1652 ;; Conditional instructions.  These are comparisons with conditional moves for
1653 ;; vectors.  They perform the assignment:
1654 ;;
1655 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1656 ;;
1657 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1658 ;; element-wise.
1659
1660 (define_expand "vcond<mode><mode>"
1661   [(set (match_operand:VDQW 0 "s_register_operand" "")
1662         (if_then_else:VDQW
1663           (match_operator 3 "comparison_operator"
1664             [(match_operand:VDQW 4 "s_register_operand" "")
1665              (match_operand:VDQW 5 "nonmemory_operand" "")])
1666           (match_operand:VDQW 1 "s_register_operand" "")
1667           (match_operand:VDQW 2 "s_register_operand" "")))]
1668   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1669 {
1670   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1671                              ? 3 : 1;
1672   rtx magic_rtx = GEN_INT (magic_word);
1673   int inverse = 0;
1674   int swap_bsl_operands = 0;
1675   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1676   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1677
1678   rtx (*base_comparison) (rtx, rtx, rtx, rtx);
1679   rtx (*complimentary_comparison) (rtx, rtx, rtx, rtx);
1680
1681   switch (GET_CODE (operands[3]))
1682     {
1683     case GE:
1684     case LE:
1685     case EQ:
1686       if (!REG_P (operands[5])
1687           && (operands[5] != CONST0_RTX (<MODE>mode)))
1688         operands[5] = force_reg (<MODE>mode, operands[5]);
1689       break;
1690     default:
1691       if (!REG_P (operands[5]))
1692         operands[5] = force_reg (<MODE>mode, operands[5]);
1693     }
1694
1695   switch (GET_CODE (operands[3]))
1696     {
1697     case LT:
1698     case UNLT:
1699       inverse = 1;
1700       /* Fall through.  */
1701     case GE:
1702     case UNGE:
1703     case ORDERED:
1704     case UNORDERED:
1705       base_comparison = gen_neon_vcge<mode>;
1706       complimentary_comparison = gen_neon_vcgt<mode>;
1707       break;
1708     case LE:
1709     case UNLE:
1710       inverse = 1;
1711       /* Fall through.  */
1712     case GT:
1713     case UNGT:
1714       base_comparison = gen_neon_vcgt<mode>;
1715       complimentary_comparison = gen_neon_vcge<mode>;
1716       break;
1717     case EQ:
1718     case NE:
1719     case UNEQ:
1720       base_comparison = gen_neon_vceq<mode>;
1721       complimentary_comparison = gen_neon_vceq<mode>;
1722       break;
1723     default:
1724       gcc_unreachable ();
1725     }
1726
1727   switch (GET_CODE (operands[3]))
1728     {
1729     case LT:
1730     case LE:
1731     case GT:
1732     case GE:
1733     case EQ:
1734       /* The easy case.  Here we emit one of vcge, vcgt or vceq.
1735          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1736          a GE b -> a GE b
1737          a GT b -> a GT b
1738          a LE b -> b GE a
1739          a LT b -> b GT a
1740          a EQ b -> a EQ b  */
1741
1742       if (!inverse)
1743         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1744       else
1745         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1746       break;
1747     case UNLT:
1748     case UNLE:
1749     case UNGT:
1750     case UNGE:
1751     case NE:
1752       /* Vector compare returns false for lanes which are unordered, so if we use
1753          the inverse of the comparison we actually want to emit, then
1754          swap the operands to BSL, we will end up with the correct result.
1755          Note that a NE NaN and NaN NE b are true for all a, b.
1756
1757          Our transformations are:
1758          a GE b -> !(b GT a)
1759          a GT b -> !(b GE a)
1760          a LE b -> !(a GT b)
1761          a LT b -> !(a GE b)
1762          a NE b -> !(a EQ b)  */
1763
1764       if (inverse)
1765         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1766       else
1767         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1768
1769       swap_bsl_operands = 1;
1770       break;
1771     case UNEQ:
1772       /* We check (a > b ||  b > a).  combining these comparisons give us
1773          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1774          will then give us (a == b ||  a UNORDERED b) as intended.  */
1775
1776       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], magic_rtx));
1777       emit_insn (gen_neon_vcgt<mode> (tmp, operands[5], operands[4], magic_rtx));
1778       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1779       swap_bsl_operands = 1;
1780       break;
1781     case UNORDERED:
1782        /* Operands are ORDERED iff (a > b || b >= a).
1783          Swapping the operands to BSL will give the UNORDERED case.  */
1784      swap_bsl_operands = 1;
1785      /* Fall through.  */
1786     case ORDERED:
1787       emit_insn (gen_neon_vcgt<mode> (tmp, operands[4], operands[5], magic_rtx));
1788       emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], magic_rtx));
1789       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1790       break;
1791     default:
1792       gcc_unreachable ();
1793     }
1794
1795   if (swap_bsl_operands)
1796     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1797                                     operands[1]));
1798   else
1799     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1800                                     operands[2]));
1801   DONE;
1802 })
1803
1804 (define_expand "vcondu<mode><mode>"
1805   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1806         (if_then_else:VDQIW
1807           (match_operator 3 "arm_comparison_operator"
1808             [(match_operand:VDQIW 4 "s_register_operand" "")
1809              (match_operand:VDQIW 5 "s_register_operand" "")])
1810           (match_operand:VDQIW 1 "s_register_operand" "")
1811           (match_operand:VDQIW 2 "s_register_operand" "")))]
1812   "TARGET_NEON"
1813 {
1814   rtx mask;
1815   int inverse = 0, immediate_zero = 0;
1816
1817   mask = gen_reg_rtx (<V_cmp_result>mode);
1818
1819   if (operands[5] == CONST0_RTX (<MODE>mode))
1820     immediate_zero = 1;
1821   else if (!REG_P (operands[5]))
1822     operands[5] = force_reg (<MODE>mode, operands[5]);
1823
1824   switch (GET_CODE (operands[3]))
1825     {
1826     case GEU:
1827       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1828                                       const0_rtx));
1829       break;
1830
1831     case GTU:
1832       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1833                                       const0_rtx));
1834       break;
1835
1836     case EQ:
1837       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1838                                       const0_rtx));
1839       break;
1840
1841     case LEU:
1842       if (immediate_zero)
1843         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1844                                         const0_rtx));
1845       else
1846         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1847                                         const0_rtx));
1848       break;
1849
1850     case LTU:
1851       if (immediate_zero)
1852         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1853                                         const0_rtx));
1854       else
1855         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1856                                         const0_rtx));
1857       break;
1858
1859     case NE:
1860       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1861                                       const0_rtx));
1862       inverse = 1;
1863       break;
1864
1865     default:
1866       gcc_unreachable ();
1867     }
1868
1869   if (inverse)
1870     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1871                                     operands[1]));
1872   else
1873     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1874                                     operands[2]));
1875
1876   DONE;
1877 })
1878
1879 ;; Patterns for builtins.
1880
1881 ; good for plain vadd, vaddq.
1882
1883 (define_expand "neon_vadd<mode>"
1884   [(match_operand:VDQX 0 "s_register_operand" "=w")
1885    (match_operand:VDQX 1 "s_register_operand" "w")
1886    (match_operand:VDQX 2 "s_register_operand" "w")
1887    (match_operand:SI 3 "immediate_operand" "i")]
1888   "TARGET_NEON"
1889 {
1890   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1891     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1892   else
1893     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1894                                            operands[2]));
1895   DONE;
1896 })
1897
1898 ; Note that NEON operations don't support the full IEEE 754 standard: in
1899 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1900 ; use those instructions for autovectorization, etc. unless
1901 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1902 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1903 ; header) must work in either case: if -funsafe-math-optimizations is given,
1904 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1905 ; expand to unspecs (which may potentially limit the extent to which they might
1906 ; be optimized by generic code).
1907
1908 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1909
1910 (define_insn "neon_vadd<mode>_unspec"
1911   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1912         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1913                       (match_operand:VDQX 2 "s_register_operand" "w")]
1914                      UNSPEC_VADD))]
1915   "TARGET_NEON"
1916   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1917   [(set (attr "neon_type")
1918       (if_then_else (match_test "<Is_float_mode>")
1919                     (if_then_else (match_test "<Is_d_reg>")
1920                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1921                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1922                     (const_string "neon_int_1")))]
1923 )
1924
1925 ; operand 3 represents in bits:
1926 ;  bit 0: signed (vs unsigned).
1927 ;  bit 1: rounding (vs none).
1928
1929 (define_insn "neon_vaddl<mode>"
1930   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1931         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1932                            (match_operand:VDI 2 "s_register_operand" "w")
1933                            (match_operand:SI 3 "immediate_operand" "i")]
1934                           UNSPEC_VADDL))]
1935   "TARGET_NEON"
1936   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1937   [(set_attr "neon_type" "neon_int_3")]
1938 )
1939
1940 (define_insn "neon_vaddw<mode>"
1941   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1942         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1943                            (match_operand:VDI 2 "s_register_operand" "w")
1944                            (match_operand:SI 3 "immediate_operand" "i")]
1945                           UNSPEC_VADDW))]
1946   "TARGET_NEON"
1947   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1948   [(set_attr "neon_type" "neon_int_2")]
1949 )
1950
1951 ; vhadd and vrhadd.
1952
1953 (define_insn "neon_vhadd<mode>"
1954   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1955         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1956                        (match_operand:VDQIW 2 "s_register_operand" "w")
1957                        (match_operand:SI 3 "immediate_operand" "i")]
1958                       UNSPEC_VHADD))]
1959   "TARGET_NEON"
1960   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1961   [(set_attr "neon_type" "neon_int_4")]
1962 )
1963
1964 (define_insn "neon_vqadd<mode>"
1965   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1966         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1967                        (match_operand:VDQIX 2 "s_register_operand" "w")
1968                        (match_operand:SI 3 "immediate_operand" "i")]
1969                      UNSPEC_VQADD))]
1970   "TARGET_NEON"
1971   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1972   [(set_attr "neon_type" "neon_int_4")]
1973 )
1974
1975 (define_insn "neon_vaddhn<mode>"
1976   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1977         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1978                             (match_operand:VN 2 "s_register_operand" "w")
1979                             (match_operand:SI 3 "immediate_operand" "i")]
1980                            UNSPEC_VADDHN))]
1981   "TARGET_NEON"
1982   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1983   [(set_attr "neon_type" "neon_int_4")]
1984 )
1985
1986 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1987 ;; polynomial multiplication case that can specified by operand 3.
1988 (define_insn "neon_vmul<mode>"
1989   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1990         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1991                       (match_operand:VDQW 2 "s_register_operand" "w")
1992                       (match_operand:SI 3 "immediate_operand" "i")]
1993                      UNSPEC_VMUL))]
1994   "TARGET_NEON"
1995   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1996   [(set (attr "neon_type")
1997       (if_then_else (match_test "<Is_float_mode>")
1998                     (if_then_else (match_test "<Is_d_reg>")
1999                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2000                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2001                     (if_then_else (match_test "<Is_d_reg>")
2002                                   (if_then_else
2003                                     (match_test "<Scalar_mul_8_16>")
2004                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2005                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2006                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2007                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
2008                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
2009 )
2010
2011 (define_expand "neon_vmla<mode>"
2012   [(match_operand:VDQW 0 "s_register_operand" "=w")
2013    (match_operand:VDQW 1 "s_register_operand" "0")
2014    (match_operand:VDQW 2 "s_register_operand" "w")
2015    (match_operand:VDQW 3 "s_register_operand" "w")
2016    (match_operand:SI 4 "immediate_operand" "i")]
2017   "TARGET_NEON"
2018 {
2019   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2020     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
2021                                              operands[2], operands[3]));
2022   else
2023     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
2024                                            operands[2], operands[3]));
2025   DONE;
2026 })
2027
2028 (define_expand "neon_vfma<VCVTF:mode>"
2029   [(match_operand:VCVTF 0 "s_register_operand")
2030    (match_operand:VCVTF 1 "s_register_operand")
2031    (match_operand:VCVTF 2 "s_register_operand")
2032    (match_operand:VCVTF 3 "s_register_operand")
2033    (match_operand:SI 4 "immediate_operand")]
2034   "TARGET_NEON && TARGET_FMA"
2035 {
2036   emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3],
2037                                        operands[1]));
2038   DONE;
2039 })
2040
2041 (define_expand "neon_vfms<VCVTF:mode>"
2042   [(match_operand:VCVTF 0 "s_register_operand")
2043    (match_operand:VCVTF 1 "s_register_operand")
2044    (match_operand:VCVTF 2 "s_register_operand")
2045    (match_operand:VCVTF 3 "s_register_operand")
2046    (match_operand:SI 4 "immediate_operand")]
2047   "TARGET_NEON && TARGET_FMA"
2048 {
2049   emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3],
2050                                          operands[1]));
2051   DONE;
2052 })
2053
2054 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2055
2056 (define_insn "neon_vmla<mode>_unspec"
2057   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2058         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2059                      (match_operand:VDQ 2 "s_register_operand" "w")
2060                      (match_operand:VDQ 3 "s_register_operand" "w")]
2061                     UNSPEC_VMLA))]
2062   "TARGET_NEON"
2063   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2064   [(set (attr "neon_type")
2065       (if_then_else (match_test "<Is_float_mode>")
2066                     (if_then_else (match_test "<Is_d_reg>")
2067                                   (const_string "neon_fp_vmla_ddd")
2068                                   (const_string "neon_fp_vmla_qqq"))
2069                     (if_then_else (match_test "<Is_d_reg>")
2070                                   (if_then_else
2071                                     (match_test "<Scalar_mul_8_16>")
2072                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2073                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2074                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2075                                     (const_string "neon_mla_qqq_8_16")
2076                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2077 )
2078
2079 (define_insn "neon_vmlal<mode>"
2080   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2081         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2082                            (match_operand:VW 2 "s_register_operand" "w")
2083                            (match_operand:VW 3 "s_register_operand" "w")
2084                            (match_operand:SI 4 "immediate_operand" "i")]
2085                           UNSPEC_VMLAL))]
2086   "TARGET_NEON"
2087   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2088   [(set (attr "neon_type")
2089      (if_then_else (match_test "<Scalar_mul_8_16>")
2090                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2091                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2092 )
2093
2094 (define_expand "neon_vmls<mode>"
2095   [(match_operand:VDQW 0 "s_register_operand" "=w")
2096    (match_operand:VDQW 1 "s_register_operand" "0")
2097    (match_operand:VDQW 2 "s_register_operand" "w")
2098    (match_operand:VDQW 3 "s_register_operand" "w")
2099    (match_operand:SI 4 "immediate_operand" "i")]
2100   "TARGET_NEON"
2101 {
2102   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2103     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
2104                  operands[1], operands[2], operands[3]));
2105   else
2106     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
2107                                            operands[2], operands[3]));
2108   DONE;
2109 })
2110
2111 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2112
2113 (define_insn "neon_vmls<mode>_unspec"
2114   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2115         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2116                      (match_operand:VDQ 2 "s_register_operand" "w")
2117                      (match_operand:VDQ 3 "s_register_operand" "w")]
2118                     UNSPEC_VMLS))]
2119   "TARGET_NEON"
2120   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2121   [(set (attr "neon_type")
2122       (if_then_else (match_test "<Is_float_mode>")
2123                     (if_then_else (match_test "<Is_d_reg>")
2124                                   (const_string "neon_fp_vmla_ddd")
2125                                   (const_string "neon_fp_vmla_qqq"))
2126                     (if_then_else (match_test "<Is_d_reg>")
2127                                   (if_then_else
2128                                     (match_test "<Scalar_mul_8_16>")
2129                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2130                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2131                                   (if_then_else
2132                                     (match_test "<Scalar_mul_8_16>")
2133                                     (const_string "neon_mla_qqq_8_16")
2134                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2135 )
2136
2137 (define_insn "neon_vmlsl<mode>"
2138   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2139         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2140                            (match_operand:VW 2 "s_register_operand" "w")
2141                            (match_operand:VW 3 "s_register_operand" "w")
2142                            (match_operand:SI 4 "immediate_operand" "i")]
2143                           UNSPEC_VMLSL))]
2144   "TARGET_NEON"
2145   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2146   [(set (attr "neon_type")
2147      (if_then_else (match_test "<Scalar_mul_8_16>")
2148                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2149                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2150 )
2151
2152 (define_insn "neon_vqdmulh<mode>"
2153   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2154         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2155                        (match_operand:VMDQI 2 "s_register_operand" "w")
2156                        (match_operand:SI 3 "immediate_operand" "i")]
2157                       UNSPEC_VQDMULH))]
2158   "TARGET_NEON"
2159   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2160   [(set (attr "neon_type")
2161      (if_then_else (match_test "<Is_d_reg>")
2162         (if_then_else (match_test "<Scalar_mul_8_16>")
2163                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2164                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2165         (if_then_else (match_test "<Scalar_mul_8_16>")
2166                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2167                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2168 )
2169
2170 (define_insn "neon_vqdmlal<mode>"
2171   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2172         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2173                            (match_operand:VMDI 2 "s_register_operand" "w")
2174                            (match_operand:VMDI 3 "s_register_operand" "w")
2175                            (match_operand:SI 4 "immediate_operand" "i")]
2176                           UNSPEC_VQDMLAL))]
2177   "TARGET_NEON"
2178   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2179   [(set (attr "neon_type")
2180      (if_then_else (match_test "<Scalar_mul_8_16>")
2181                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2182                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2183 )
2184
2185 (define_insn "neon_vqdmlsl<mode>"
2186   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2187         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2188                            (match_operand:VMDI 2 "s_register_operand" "w")
2189                            (match_operand:VMDI 3 "s_register_operand" "w")
2190                            (match_operand:SI 4 "immediate_operand" "i")]
2191                           UNSPEC_VQDMLSL))]
2192   "TARGET_NEON"
2193   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2194   [(set (attr "neon_type")
2195      (if_then_else (match_test "<Scalar_mul_8_16>")
2196                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2197                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2198 )
2199
2200 (define_insn "neon_vmull<mode>"
2201   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2202         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2203                            (match_operand:VW 2 "s_register_operand" "w")
2204                            (match_operand:SI 3 "immediate_operand" "i")]
2205                           UNSPEC_VMULL))]
2206   "TARGET_NEON"
2207   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2208   [(set (attr "neon_type")
2209      (if_then_else (match_test "<Scalar_mul_8_16>")
2210                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2211                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2212 )
2213
2214 (define_insn "neon_vqdmull<mode>"
2215   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2216         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2217                            (match_operand:VMDI 2 "s_register_operand" "w")
2218                            (match_operand:SI 3 "immediate_operand" "i")]
2219                           UNSPEC_VQDMULL))]
2220   "TARGET_NEON"
2221   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2222   [(set (attr "neon_type")
2223      (if_then_else (match_test "<Scalar_mul_8_16>")
2224                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2225                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2226 )
2227
2228 (define_expand "neon_vsub<mode>"
2229   [(match_operand:VDQX 0 "s_register_operand" "=w")
2230    (match_operand:VDQX 1 "s_register_operand" "w")
2231    (match_operand:VDQX 2 "s_register_operand" "w")
2232    (match_operand:SI 3 "immediate_operand" "i")]
2233   "TARGET_NEON"
2234 {
2235   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2236     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2237   else
2238     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2239                                            operands[2]));
2240   DONE;
2241 })
2242
2243 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2244
2245 (define_insn "neon_vsub<mode>_unspec"
2246   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2247         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2248                       (match_operand:VDQX 2 "s_register_operand" "w")]
2249                      UNSPEC_VSUB))]
2250   "TARGET_NEON"
2251   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2252   [(set (attr "neon_type")
2253       (if_then_else (match_test "<Is_float_mode>")
2254                     (if_then_else (match_test "<Is_d_reg>")
2255                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2256                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2257                     (const_string "neon_int_2")))]
2258 )
2259
2260 (define_insn "neon_vsubl<mode>"
2261   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2262         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2263                            (match_operand:VDI 2 "s_register_operand" "w")
2264                            (match_operand:SI 3 "immediate_operand" "i")]
2265                           UNSPEC_VSUBL))]
2266   "TARGET_NEON"
2267   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2268   [(set_attr "neon_type" "neon_int_2")]
2269 )
2270
2271 (define_insn "neon_vsubw<mode>"
2272   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2273         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2274                            (match_operand:VDI 2 "s_register_operand" "w")
2275                            (match_operand:SI 3 "immediate_operand" "i")]
2276                           UNSPEC_VSUBW))]
2277   "TARGET_NEON"
2278   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2279   [(set_attr "neon_type" "neon_int_2")]
2280 )
2281
2282 (define_insn "neon_vqsub<mode>"
2283   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2284         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2285                        (match_operand:VDQIX 2 "s_register_operand" "w")
2286                        (match_operand:SI 3 "immediate_operand" "i")]
2287                       UNSPEC_VQSUB))]
2288   "TARGET_NEON"
2289   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2290   [(set_attr "neon_type" "neon_int_5")]
2291 )
2292
2293 (define_insn "neon_vhsub<mode>"
2294   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2295         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2296                        (match_operand:VDQIW 2 "s_register_operand" "w")
2297                        (match_operand:SI 3 "immediate_operand" "i")]
2298                       UNSPEC_VHSUB))]
2299   "TARGET_NEON"
2300   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2301   [(set_attr "neon_type" "neon_int_5")]
2302 )
2303
2304 (define_insn "neon_vsubhn<mode>"
2305   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2306         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2307                             (match_operand:VN 2 "s_register_operand" "w")
2308                             (match_operand:SI 3 "immediate_operand" "i")]
2309                            UNSPEC_VSUBHN))]
2310   "TARGET_NEON"
2311   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2312   [(set_attr "neon_type" "neon_int_4")]
2313 )
2314
2315 (define_insn "neon_vceq<mode>"
2316   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2317         (unspec:<V_cmp_result>
2318           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2319            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2320            (match_operand:SI 3 "immediate_operand" "i,i")]
2321           UNSPEC_VCEQ))]
2322   "TARGET_NEON"
2323   "@
2324   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2325   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2326   [(set (attr "neon_type")
2327       (if_then_else (match_test "<Is_float_mode>")
2328                     (if_then_else (match_test "<Is_d_reg>")
2329                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2330                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2331                     (const_string "neon_int_5")))]
2332 )
2333
2334 (define_insn "neon_vcge<mode>"
2335   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2336         (unspec:<V_cmp_result>
2337           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2338            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2339            (match_operand:SI 3 "immediate_operand" "i,i")]
2340           UNSPEC_VCGE))]
2341   "TARGET_NEON"
2342   "@
2343   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2344   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2345   [(set (attr "neon_type")
2346      (if_then_else (match_test "<Is_float_mode>")
2347                    (if_then_else (match_test "<Is_d_reg>")
2348                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2349                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2350                    (const_string "neon_int_5")))]
2351 )
2352
2353 (define_insn "neon_vcgeu<mode>"
2354   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2355         (unspec:<V_cmp_result>
2356           [(match_operand:VDQIW 1 "s_register_operand" "w")
2357            (match_operand:VDQIW 2 "s_register_operand" "w")
2358            (match_operand:SI 3 "immediate_operand" "i")]
2359           UNSPEC_VCGEU))]
2360   "TARGET_NEON"
2361   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2362   [(set_attr "neon_type" "neon_int_5")]
2363 )
2364
2365 (define_insn "neon_vcgt<mode>"
2366   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2367         (unspec:<V_cmp_result>
2368           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2369            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2370            (match_operand:SI 3 "immediate_operand" "i,i")]
2371           UNSPEC_VCGT))]
2372   "TARGET_NEON"
2373   "@
2374   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2375   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2376   [(set (attr "neon_type")
2377      (if_then_else (match_test "<Is_float_mode>")
2378                    (if_then_else (match_test "<Is_d_reg>")
2379                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2380                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2381                    (const_string "neon_int_5")))]
2382 )
2383
2384 (define_insn "neon_vcgtu<mode>"
2385   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2386         (unspec:<V_cmp_result>
2387           [(match_operand:VDQIW 1 "s_register_operand" "w")
2388            (match_operand:VDQIW 2 "s_register_operand" "w")
2389            (match_operand:SI 3 "immediate_operand" "i")]
2390           UNSPEC_VCGTU))]
2391   "TARGET_NEON"
2392   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2393   [(set_attr "neon_type" "neon_int_5")]
2394 )
2395
2396 ;; VCLE and VCLT only support comparisons with immediate zero (register
2397 ;; variants are VCGE and VCGT with operands reversed).
2398
2399 (define_insn "neon_vcle<mode>"
2400   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2401         (unspec:<V_cmp_result>
2402           [(match_operand:VDQW 1 "s_register_operand" "w")
2403            (match_operand:VDQW 2 "zero_operand" "Dz")
2404            (match_operand:SI 3 "immediate_operand" "i")]
2405           UNSPEC_VCLE))]
2406   "TARGET_NEON"
2407   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2408   [(set (attr "neon_type")
2409       (if_then_else (match_test "<Is_float_mode>")
2410                     (if_then_else (match_test "<Is_d_reg>")
2411                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2412                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2413                     (const_string "neon_int_5")))]
2414 )
2415
2416 (define_insn "neon_vclt<mode>"
2417   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2418         (unspec:<V_cmp_result>
2419           [(match_operand:VDQW 1 "s_register_operand" "w")
2420            (match_operand:VDQW 2 "zero_operand" "Dz")
2421            (match_operand:SI 3 "immediate_operand" "i")]
2422           UNSPEC_VCLT))]
2423   "TARGET_NEON"
2424   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2425   [(set (attr "neon_type")
2426       (if_then_else (match_test "<Is_float_mode>")
2427                     (if_then_else (match_test "<Is_d_reg>")
2428                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2429                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2430                     (const_string "neon_int_5")))]
2431 )
2432
2433 (define_insn "neon_vcage<mode>"
2434   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2435         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2436                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2437                                 (match_operand:SI 3 "immediate_operand" "i")]
2438                                UNSPEC_VCAGE))]
2439   "TARGET_NEON"
2440   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2441   [(set (attr "neon_type")
2442      (if_then_else (match_test "<Is_d_reg>")
2443                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2444                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2445 )
2446
2447 (define_insn "neon_vcagt<mode>"
2448   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2449         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2450                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2451                                 (match_operand:SI 3 "immediate_operand" "i")]
2452                                UNSPEC_VCAGT))]
2453   "TARGET_NEON"
2454   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2455   [(set (attr "neon_type")
2456      (if_then_else (match_test "<Is_d_reg>")
2457                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2458                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2459 )
2460
2461 (define_insn "neon_vtst<mode>"
2462   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2463         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2464                        (match_operand:VDQIW 2 "s_register_operand" "w")
2465                        (match_operand:SI 3 "immediate_operand" "i")]
2466                       UNSPEC_VTST))]
2467   "TARGET_NEON"
2468   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2469   [(set_attr "neon_type" "neon_int_4")]
2470 )
2471
2472 (define_insn "neon_vabd<mode>"
2473   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2474         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2475                       (match_operand:VDQW 2 "s_register_operand" "w")
2476                       (match_operand:SI 3 "immediate_operand" "i")]
2477                      UNSPEC_VABD))]
2478   "TARGET_NEON"
2479   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2480   [(set (attr "neon_type")
2481      (if_then_else (match_test "<Is_float_mode>")
2482                    (if_then_else (match_test "<Is_d_reg>")
2483                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2484                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2485                    (const_string "neon_int_5")))]
2486 )
2487
2488 (define_insn "neon_vabdl<mode>"
2489   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2490         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2491                            (match_operand:VW 2 "s_register_operand" "w")
2492                            (match_operand:SI 3 "immediate_operand" "i")]
2493                           UNSPEC_VABDL))]
2494   "TARGET_NEON"
2495   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2496   [(set_attr "neon_type" "neon_int_5")]
2497 )
2498
2499 (define_insn "neon_vaba<mode>"
2500   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2501         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2502                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2503                                    (match_operand:SI 4 "immediate_operand" "i")]
2504                                   UNSPEC_VABD)
2505                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2506   "TARGET_NEON"
2507   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2508   [(set (attr "neon_type")
2509      (if_then_else (match_test "<Is_d_reg>")
2510                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2511 )
2512
2513 (define_insn "neon_vabal<mode>"
2514   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2515         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2516                                            (match_operand:VW 3 "s_register_operand" "w")
2517                                            (match_operand:SI 4 "immediate_operand" "i")]
2518                                            UNSPEC_VABDL)
2519                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2520   "TARGET_NEON"
2521   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2522   [(set_attr "neon_type" "neon_vaba")]
2523 )
2524
2525 (define_insn "neon_vmax<mode>"
2526   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2527         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2528                       (match_operand:VDQW 2 "s_register_operand" "w")
2529                       (match_operand:SI 3 "immediate_operand" "i")]
2530                      UNSPEC_VMAX))]
2531   "TARGET_NEON"
2532   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2533   [(set (attr "neon_type")
2534     (if_then_else (match_test "<Is_float_mode>")
2535                   (if_then_else (match_test "<Is_d_reg>")
2536                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2537                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2538                   (const_string "neon_int_5")))]
2539 )
2540
2541 (define_insn "neon_vmin<mode>"
2542   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2543         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2544                       (match_operand:VDQW 2 "s_register_operand" "w")
2545                       (match_operand:SI 3 "immediate_operand" "i")]
2546                      UNSPEC_VMIN))]
2547   "TARGET_NEON"
2548   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2549   [(set (attr "neon_type")
2550     (if_then_else (match_test "<Is_float_mode>")
2551                   (if_then_else (match_test "<Is_d_reg>")
2552                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2553                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2554                   (const_string "neon_int_5")))]
2555 )
2556
2557 (define_expand "neon_vpadd<mode>"
2558   [(match_operand:VD 0 "s_register_operand" "=w")
2559    (match_operand:VD 1 "s_register_operand" "w")
2560    (match_operand:VD 2 "s_register_operand" "w")
2561    (match_operand:SI 3 "immediate_operand" "i")]
2562   "TARGET_NEON"
2563 {
2564   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2565                                             operands[2]));
2566   DONE;
2567 })
2568
2569 (define_insn "neon_vpaddl<mode>"
2570   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2571         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2572                                   (match_operand:SI 2 "immediate_operand" "i")]
2573                                  UNSPEC_VPADDL))]
2574   "TARGET_NEON"
2575   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2576   ;; Assume this schedules like vaddl.
2577   [(set_attr "neon_type" "neon_int_3")]
2578 )
2579
2580 (define_insn "neon_vpadal<mode>"
2581   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2582         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2583                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2584                                   (match_operand:SI 3 "immediate_operand" "i")]
2585                                  UNSPEC_VPADAL))]
2586   "TARGET_NEON"
2587   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2588   ;; Assume this schedules like vpadd.
2589   [(set_attr "neon_type" "neon_int_1")]
2590 )
2591
2592 (define_insn "neon_vpmax<mode>"
2593   [(set (match_operand:VD 0 "s_register_operand" "=w")
2594         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2595                     (match_operand:VD 2 "s_register_operand" "w")
2596                     (match_operand:SI 3 "immediate_operand" "i")]
2597                    UNSPEC_VPMAX))]
2598   "TARGET_NEON"
2599   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2600   ;; Assume this schedules like vmax.
2601   [(set (attr "neon_type")
2602     (if_then_else (match_test "<Is_float_mode>")
2603                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2604                   (const_string "neon_int_5")))]
2605 )
2606
2607 (define_insn "neon_vpmin<mode>"
2608   [(set (match_operand:VD 0 "s_register_operand" "=w")
2609         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2610                     (match_operand:VD 2 "s_register_operand" "w")
2611                     (match_operand:SI 3 "immediate_operand" "i")]
2612                    UNSPEC_VPMIN))]
2613   "TARGET_NEON"
2614   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2615   ;; Assume this schedules like vmin.
2616   [(set (attr "neon_type")
2617     (if_then_else (match_test "<Is_float_mode>")
2618                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2619                   (const_string "neon_int_5")))]
2620 )
2621
2622 (define_insn "neon_vrecps<mode>"
2623   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2624         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2625                        (match_operand:VCVTF 2 "s_register_operand" "w")
2626                        (match_operand:SI 3 "immediate_operand" "i")]
2627                       UNSPEC_VRECPS))]
2628   "TARGET_NEON"
2629   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2630   [(set (attr "neon_type")
2631       (if_then_else (match_test "<Is_d_reg>")
2632                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2633                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2634 )
2635
2636 (define_insn "neon_vrsqrts<mode>"
2637   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2638         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2639                        (match_operand:VCVTF 2 "s_register_operand" "w")
2640                        (match_operand:SI 3 "immediate_operand" "i")]
2641                       UNSPEC_VRSQRTS))]
2642   "TARGET_NEON"
2643   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2644   [(set (attr "neon_type")
2645       (if_then_else (match_test "<Is_d_reg>")
2646                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2647                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2648 )
2649
2650 (define_expand "neon_vabs<mode>"
2651   [(match_operand:VDQW 0 "s_register_operand" "")
2652    (match_operand:VDQW 1 "s_register_operand" "")
2653    (match_operand:SI 2 "immediate_operand" "")]
2654   "TARGET_NEON"
2655 {
2656   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2657   DONE;
2658 })
2659
2660 (define_insn "neon_vqabs<mode>"
2661   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2662         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2663                        (match_operand:SI 2 "immediate_operand" "i")]
2664                       UNSPEC_VQABS))]
2665   "TARGET_NEON"
2666   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2667   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2668 )
2669
2670 (define_expand "neon_vneg<mode>"
2671   [(match_operand:VDQW 0 "s_register_operand" "")
2672    (match_operand:VDQW 1 "s_register_operand" "")
2673    (match_operand:SI 2 "immediate_operand" "")]
2674   "TARGET_NEON"
2675 {
2676   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2677   DONE;
2678 })
2679
2680 (define_insn "neon_vqneg<mode>"
2681   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2682         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2683                        (match_operand:SI 2 "immediate_operand" "i")]
2684                       UNSPEC_VQNEG))]
2685   "TARGET_NEON"
2686   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2687   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2688 )
2689
2690 (define_insn "neon_vcls<mode>"
2691   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2692         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2693                        (match_operand:SI 2 "immediate_operand" "i")]
2694                       UNSPEC_VCLS))]
2695   "TARGET_NEON"
2696   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2697   [(set_attr "neon_type" "neon_int_1")]
2698 )
2699
2700 (define_insn "clz<mode>2"
2701   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2702         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2703   "TARGET_NEON"
2704   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2705   [(set_attr "neon_type" "neon_int_1")]
2706 )
2707
2708 (define_expand "neon_vclz<mode>"
2709   [(match_operand:VDQIW 0 "s_register_operand" "")
2710    (match_operand:VDQIW 1 "s_register_operand" "")
2711    (match_operand:SI 2 "immediate_operand" "")]
2712   "TARGET_NEON"
2713 {
2714   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2715   DONE;
2716 })
2717
2718 (define_insn "popcount<mode>2"
2719   [(set (match_operand:VE 0 "s_register_operand" "=w")
2720         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2721   "TARGET_NEON"
2722   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2723   [(set_attr "neon_type" "neon_int_1")]
2724 )
2725
2726 (define_expand "neon_vcnt<mode>"
2727   [(match_operand:VE 0 "s_register_operand" "=w")
2728    (match_operand:VE 1 "s_register_operand" "w")
2729    (match_operand:SI 2 "immediate_operand" "i")]
2730   "TARGET_NEON"
2731 {
2732   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2733   DONE;
2734 })
2735
2736 (define_insn "neon_vrecpe<mode>"
2737   [(set (match_operand:V32 0 "s_register_operand" "=w")
2738         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2739                      (match_operand:SI 2 "immediate_operand" "i")]
2740                     UNSPEC_VRECPE))]
2741   "TARGET_NEON"
2742   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2743   [(set (attr "neon_type")
2744       (if_then_else (match_test "<Is_d_reg>")
2745                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2746                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2747 )
2748
2749 (define_insn "neon_vrsqrte<mode>"
2750   [(set (match_operand:V32 0 "s_register_operand" "=w")
2751         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2752                      (match_operand:SI 2 "immediate_operand" "i")]
2753                     UNSPEC_VRSQRTE))]
2754   "TARGET_NEON"
2755   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2756   [(set (attr "neon_type")
2757       (if_then_else (match_test "<Is_d_reg>")
2758                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2759                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2760 )
2761
2762 (define_expand "neon_vmvn<mode>"
2763   [(match_operand:VDQIW 0 "s_register_operand" "")
2764    (match_operand:VDQIW 1 "s_register_operand" "")
2765    (match_operand:SI 2 "immediate_operand" "")]
2766   "TARGET_NEON"
2767 {
2768   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2769   DONE;
2770 })
2771
2772 (define_insn "neon_vget_lane<mode>_sext_internal"
2773   [(set (match_operand:SI 0 "s_register_operand" "=r")
2774         (sign_extend:SI
2775           (vec_select:<V_elem>
2776             (match_operand:VD 1 "s_register_operand" "w")
2777             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2778   "TARGET_NEON"
2779 {
2780   if (BYTES_BIG_ENDIAN)
2781     {
2782       int elt = INTVAL (operands[2]);
2783       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2784       operands[2] = GEN_INT (elt);
2785     }
2786   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2787 }
2788   [(set_attr "neon_type" "neon_bp_simple")]
2789 )
2790
2791 (define_insn "neon_vget_lane<mode>_zext_internal"
2792   [(set (match_operand:SI 0 "s_register_operand" "=r")
2793         (zero_extend:SI
2794           (vec_select:<V_elem>
2795             (match_operand:VD 1 "s_register_operand" "w")
2796             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2797   "TARGET_NEON"
2798 {
2799   if (BYTES_BIG_ENDIAN)
2800     {
2801       int elt = INTVAL (operands[2]);
2802       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2803       operands[2] = GEN_INT (elt);
2804     }
2805   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2806 }
2807   [(set_attr "neon_type" "neon_bp_simple")]
2808 )
2809
2810 (define_insn "neon_vget_lane<mode>_sext_internal"
2811   [(set (match_operand:SI 0 "s_register_operand" "=r")
2812         (sign_extend:SI
2813           (vec_select:<V_elem>
2814             (match_operand:VQ 1 "s_register_operand" "w")
2815             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2816   "TARGET_NEON"
2817 {
2818   rtx ops[3];
2819   int regno = REGNO (operands[1]);
2820   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2821   unsigned int elt = INTVAL (operands[2]);
2822   unsigned int elt_adj = elt % halfelts;
2823
2824   if (BYTES_BIG_ENDIAN)
2825     elt_adj = halfelts - 1 - elt_adj;
2826
2827   ops[0] = operands[0];
2828   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2829   ops[2] = GEN_INT (elt_adj);
2830   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2831
2832   return "";
2833 }
2834   [(set_attr "neon_type" "neon_bp_simple")]
2835 )
2836
2837 (define_insn "neon_vget_lane<mode>_zext_internal"
2838   [(set (match_operand:SI 0 "s_register_operand" "=r")
2839         (zero_extend:SI
2840           (vec_select:<V_elem>
2841             (match_operand:VQ 1 "s_register_operand" "w")
2842             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2843   "TARGET_NEON"
2844 {
2845   rtx ops[3];
2846   int regno = REGNO (operands[1]);
2847   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2848   unsigned int elt = INTVAL (operands[2]);
2849   unsigned int elt_adj = elt % halfelts;
2850
2851   if (BYTES_BIG_ENDIAN)
2852     elt_adj = halfelts - 1 - elt_adj;
2853
2854   ops[0] = operands[0];
2855   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2856   ops[2] = GEN_INT (elt_adj);
2857   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2858
2859   return "";
2860 }
2861   [(set_attr "neon_type" "neon_bp_simple")]
2862 )
2863
2864 (define_expand "neon_vget_lane<mode>"
2865   [(match_operand:<V_ext> 0 "s_register_operand" "")
2866    (match_operand:VDQW 1 "s_register_operand" "")
2867    (match_operand:SI 2 "immediate_operand" "")
2868    (match_operand:SI 3 "immediate_operand" "")]
2869   "TARGET_NEON"
2870 {
2871   HOST_WIDE_INT magic = INTVAL (operands[3]);
2872   rtx insn;
2873
2874   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2875
2876   if (BYTES_BIG_ENDIAN)
2877     {
2878       /* The intrinsics are defined in terms of a model where the
2879          element ordering in memory is vldm order, whereas the generic
2880          RTL is defined in terms of a model where the element ordering
2881          in memory is array order.  Convert the lane number to conform
2882          to this model.  */
2883       unsigned int elt = INTVAL (operands[2]);
2884       unsigned int reg_nelts
2885         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2886       elt ^= reg_nelts - 1;
2887       operands[2] = GEN_INT (elt);
2888     }
2889
2890   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2891     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2892   else
2893     {
2894       if ((magic & 1) != 0)
2895         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2896                                                        operands[2]);
2897       else
2898         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2899                                                        operands[2]);
2900     }
2901   emit_insn (insn);
2902   DONE;
2903 })
2904
2905 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2906 ; elements.
2907
2908 (define_expand "neon_vget_lanedi"
2909   [(match_operand:DI 0 "s_register_operand" "=r")
2910    (match_operand:DI 1 "s_register_operand" "w")
2911    (match_operand:SI 2 "immediate_operand" "i")
2912    (match_operand:SI 3 "immediate_operand" "i")]
2913   "TARGET_NEON"
2914 {
2915   neon_lane_bounds (operands[2], 0, 1);
2916   emit_move_insn (operands[0], operands[1]);
2917   DONE;
2918 })
2919
2920 (define_expand "neon_vget_lanev2di"
2921   [(match_operand:DI 0 "s_register_operand" "")
2922    (match_operand:V2DI 1 "s_register_operand" "")
2923    (match_operand:SI 2 "immediate_operand" "")
2924    (match_operand:SI 3 "immediate_operand" "")]
2925   "TARGET_NEON"
2926 {
2927   switch (INTVAL (operands[2]))
2928     {
2929     case 0:
2930       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2931       break;
2932     case 1:
2933       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2934       break;
2935     default:
2936       neon_lane_bounds (operands[2], 0, 1);
2937       FAIL;
2938     }
2939   DONE;
2940 })
2941
2942 (define_expand "neon_vset_lane<mode>"
2943   [(match_operand:VDQ 0 "s_register_operand" "=w")
2944    (match_operand:<V_elem> 1 "s_register_operand" "r")
2945    (match_operand:VDQ 2 "s_register_operand" "0")
2946    (match_operand:SI 3 "immediate_operand" "i")]
2947   "TARGET_NEON"
2948 {
2949   unsigned int elt = INTVAL (operands[3]);
2950   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2951
2952   if (BYTES_BIG_ENDIAN)
2953     {
2954       unsigned int reg_nelts
2955         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2956       elt ^= reg_nelts - 1;
2957     }
2958
2959   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2960                                          GEN_INT (1 << elt), operands[2]));
2961   DONE;
2962 })
2963
2964 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2965
2966 (define_expand "neon_vset_lanedi"
2967   [(match_operand:DI 0 "s_register_operand" "=w")
2968    (match_operand:DI 1 "s_register_operand" "r")
2969    (match_operand:DI 2 "s_register_operand" "0")
2970    (match_operand:SI 3 "immediate_operand" "i")]
2971   "TARGET_NEON"
2972 {
2973   neon_lane_bounds (operands[3], 0, 1);
2974   emit_move_insn (operands[0], operands[1]);
2975   DONE;
2976 })
2977
2978 (define_expand "neon_vcreate<mode>"
2979   [(match_operand:VDX 0 "s_register_operand" "")
2980    (match_operand:DI 1 "general_operand" "")]
2981   "TARGET_NEON"
2982 {
2983   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2984   emit_move_insn (operands[0], src);
2985   DONE;
2986 })
2987
2988 (define_insn "neon_vdup_n<mode>"
2989   [(set (match_operand:VX 0 "s_register_operand" "=w")
2990         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2991   "TARGET_NEON"
2992   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2993   ;; Assume this schedules like vmov.
2994   [(set_attr "neon_type" "neon_bp_simple")]
2995 )
2996
2997 (define_insn "neon_vdup_n<mode>"
2998   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2999         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
3000   "TARGET_NEON"
3001   "@
3002   vdup.<V_sz_elem>\t%<V_reg>0, %1
3003   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
3004   ;; Assume this schedules like vmov.
3005   [(set_attr "neon_type" "neon_bp_simple")]
3006 )
3007
3008 (define_expand "neon_vdup_ndi"
3009   [(match_operand:DI 0 "s_register_operand" "=w")
3010    (match_operand:DI 1 "s_register_operand" "r")]
3011   "TARGET_NEON"
3012 {
3013   emit_move_insn (operands[0], operands[1]);
3014   DONE;
3015 }
3016 )
3017
3018 (define_insn "neon_vdup_nv2di"
3019   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
3020         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
3021   "TARGET_NEON"
3022   "@
3023   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
3024   vmov\t%e0, %P1\;vmov\t%f0, %P1"
3025   [(set_attr "length" "8")
3026    (set_attr "neon_type" "neon_bp_simple")]
3027 )
3028
3029 (define_insn "neon_vdup_lane<mode>_internal"
3030   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
3031         (vec_duplicate:VDQW
3032           (vec_select:<V_elem>
3033             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3034             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
3035   "TARGET_NEON"
3036 {
3037   if (BYTES_BIG_ENDIAN)
3038     {
3039       int elt = INTVAL (operands[2]);
3040       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
3041       operands[2] = GEN_INT (elt);
3042     }
3043   if (<Is_d_reg>)
3044     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
3045   else
3046     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
3047 }
3048   ;; Assume this schedules like vmov.
3049   [(set_attr "neon_type" "neon_bp_simple")]
3050 )
3051
3052 (define_expand "neon_vdup_lane<mode>"
3053   [(match_operand:VDQW 0 "s_register_operand" "=w")
3054    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3055    (match_operand:SI 2 "immediate_operand" "i")]
3056   "TARGET_NEON"
3057 {
3058   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
3059   if (BYTES_BIG_ENDIAN)
3060     {
3061       unsigned int elt = INTVAL (operands[2]);
3062       unsigned int reg_nelts
3063         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
3064       elt ^= reg_nelts - 1;
3065       operands[2] = GEN_INT (elt);
3066     }
3067     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
3068                                                   operands[2]));
3069     DONE;
3070 })
3071
3072 ; Scalar index is ignored, since only zero is valid here.
3073 (define_expand "neon_vdup_lanedi"
3074   [(match_operand:DI 0 "s_register_operand" "=w")
3075    (match_operand:DI 1 "s_register_operand" "w")
3076    (match_operand:SI 2 "immediate_operand" "i")]
3077   "TARGET_NEON"
3078 {
3079   neon_lane_bounds (operands[2], 0, 1);
3080   emit_move_insn (operands[0], operands[1]);
3081   DONE;
3082 })
3083
3084 ; Likewise for v2di, as the DImode second operand has only a single element.
3085 (define_expand "neon_vdup_lanev2di"
3086   [(match_operand:V2DI 0 "s_register_operand" "=w")
3087    (match_operand:DI 1 "s_register_operand" "w")
3088    (match_operand:SI 2 "immediate_operand" "i")]
3089   "TARGET_NEON"
3090 {
3091   neon_lane_bounds (operands[2], 0, 1);
3092   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
3093   DONE;
3094 })
3095
3096 ; Disabled before reload because we don't want combine doing something silly,
3097 ; but used by the post-reload expansion of neon_vcombine.
3098 (define_insn "*neon_vswp<mode>"
3099   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
3100         (match_operand:VDQX 1 "s_register_operand" "+w"))
3101    (set (match_dup 1) (match_dup 0))]
3102   "TARGET_NEON && reload_completed"
3103   "vswp\t%<V_reg>0, %<V_reg>1"
3104   [(set (attr "neon_type")
3105         (if_then_else (match_test "<Is_d_reg>")
3106                       (const_string "neon_bp_simple")
3107                       (const_string "neon_bp_2cycle")))]
3108 )
3109
3110 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
3111 ;; dest vector.
3112 ;; FIXME: A different implementation of this builtin could make it much
3113 ;; more likely that we wouldn't actually need to output anything (we could make
3114 ;; it so that the reg allocator puts things in the right places magically
3115 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
3116
3117 (define_insn_and_split "neon_vcombine<mode>"
3118   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
3119         (vec_concat:<V_DOUBLE>
3120           (match_operand:VDX 1 "s_register_operand" "w")
3121           (match_operand:VDX 2 "s_register_operand" "w")))]
3122   "TARGET_NEON"
3123   "#"
3124   "&& reload_completed"
3125   [(const_int 0)]
3126 {
3127   neon_split_vcombine (operands);
3128   DONE;
3129 })
3130
3131 (define_expand "neon_vget_high<mode>"
3132   [(match_operand:<V_HALF> 0 "s_register_operand")
3133    (match_operand:VQX 1 "s_register_operand")]
3134   "TARGET_NEON"
3135 {
3136   emit_move_insn (operands[0],
3137                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
3138                                        GET_MODE_SIZE (<V_HALF>mode)));
3139   DONE;
3140 })
3141
3142 (define_expand "neon_vget_low<mode>"
3143   [(match_operand:<V_HALF> 0 "s_register_operand")
3144    (match_operand:VQX 1 "s_register_operand")]
3145   "TARGET_NEON"
3146 {
3147   emit_move_insn (operands[0],
3148                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3149                                        <MODE>mode, 0));
3150   DONE;
3151 })
3152
3153 (define_insn "float<mode><V_cvtto>2"
3154   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3155         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3156   "TARGET_NEON && !flag_rounding_math"
3157   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3158   [(set (attr "neon_type")
3159      (if_then_else (match_test "<Is_d_reg>")
3160                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3161                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3162 )
3163
3164 (define_insn "floatuns<mode><V_cvtto>2"
3165   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3166         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3167   "TARGET_NEON && !flag_rounding_math"
3168   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3169   [(set (attr "neon_type")
3170      (if_then_else (match_test "<Is_d_reg>")
3171                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3172                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3173 )
3174
3175 (define_insn "fix_trunc<mode><V_cvtto>2"
3176   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3177         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3178   "TARGET_NEON"
3179   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3180   [(set (attr "neon_type")
3181      (if_then_else (match_test "<Is_d_reg>")
3182                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3183                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3184 )
3185
3186 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3187   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3188         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3189   "TARGET_NEON"
3190   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3191   [(set (attr "neon_type")
3192      (if_then_else (match_test "<Is_d_reg>")
3193                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3194                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3195 )
3196
3197 (define_insn "neon_vcvt<mode>"
3198   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3199         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3200                            (match_operand:SI 2 "immediate_operand" "i")]
3201                           UNSPEC_VCVT))]
3202   "TARGET_NEON"
3203   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3204   [(set (attr "neon_type")
3205      (if_then_else (match_test "<Is_d_reg>")
3206                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3207                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3208 )
3209
3210 (define_insn "neon_vcvt<mode>"
3211   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3212         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3213                            (match_operand:SI 2 "immediate_operand" "i")]
3214                           UNSPEC_VCVT))]
3215   "TARGET_NEON"
3216   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3217   [(set (attr "neon_type")
3218      (if_then_else (match_test "<Is_d_reg>")
3219                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3220                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3221 )
3222
3223 (define_insn "neon_vcvtv4sfv4hf"
3224   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
3225         (unspec:V4SF [(match_operand:V4HF 1 "s_register_operand" "w")]
3226                           UNSPEC_VCVT))]
3227   "TARGET_NEON && TARGET_FP16"
3228   "vcvt.f32.f16\t%q0, %P1"
3229   [(set_attr "neon_type" "neon_fp_vadd_ddd_vabs_dd")]
3230 )
3231
3232 (define_insn "neon_vcvtv4hfv4sf"
3233   [(set (match_operand:V4HF 0 "s_register_operand" "=w")
3234         (unspec:V4HF [(match_operand:V4SF 1 "s_register_operand" "w")]
3235                           UNSPEC_VCVT))]
3236   "TARGET_NEON && TARGET_FP16"
3237   "vcvt.f16.f32\t%P0, %q1"
3238   [(set_attr "neon_type" "neon_fp_vadd_ddd_vabs_dd")]
3239 )
3240
3241 (define_insn "neon_vcvt_n<mode>"
3242   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3243         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3244                            (match_operand:SI 2 "immediate_operand" "i")
3245                            (match_operand:SI 3 "immediate_operand" "i")]
3246                           UNSPEC_VCVT_N))]
3247   "TARGET_NEON"
3248 {
3249   neon_const_bounds (operands[2], 1, 33);
3250   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3251 }
3252   [(set (attr "neon_type")
3253      (if_then_else (match_test "<Is_d_reg>")
3254                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3255                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3256 )
3257
3258 (define_insn "neon_vcvt_n<mode>"
3259   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3260         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3261                            (match_operand:SI 2 "immediate_operand" "i")
3262                            (match_operand:SI 3 "immediate_operand" "i")]
3263                           UNSPEC_VCVT_N))]
3264   "TARGET_NEON"
3265 {
3266   neon_const_bounds (operands[2], 1, 33);
3267   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3268 }
3269   [(set (attr "neon_type")
3270      (if_then_else (match_test "<Is_d_reg>")
3271                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3272                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3273 )
3274
3275 (define_insn "neon_vmovn<mode>"
3276   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3277         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3278                             (match_operand:SI 2 "immediate_operand" "i")]
3279                            UNSPEC_VMOVN))]
3280   "TARGET_NEON"
3281   "vmovn.<V_if_elem>\t%P0, %q1"
3282   [(set_attr "neon_type" "neon_bp_simple")]
3283 )
3284
3285 (define_insn "neon_vqmovn<mode>"
3286   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3287         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3288                             (match_operand:SI 2 "immediate_operand" "i")]
3289                            UNSPEC_VQMOVN))]
3290   "TARGET_NEON"
3291   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3292   [(set_attr "neon_type" "neon_shift_2")]
3293 )
3294
3295 (define_insn "neon_vqmovun<mode>"
3296   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3297         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3298                             (match_operand:SI 2 "immediate_operand" "i")]
3299                            UNSPEC_VQMOVUN))]
3300   "TARGET_NEON"
3301   "vqmovun.<V_s_elem>\t%P0, %q1"
3302   [(set_attr "neon_type" "neon_shift_2")]
3303 )
3304
3305 (define_insn "neon_vmovl<mode>"
3306   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3307         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3308                            (match_operand:SI 2 "immediate_operand" "i")]
3309                           UNSPEC_VMOVL))]
3310   "TARGET_NEON"
3311   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3312   [(set_attr "neon_type" "neon_shift_1")]
3313 )
3314
3315 (define_insn "neon_vmul_lane<mode>"
3316   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3317         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3318                      (match_operand:VMD 2 "s_register_operand"
3319                                         "<scalar_mul_constraint>")
3320                      (match_operand:SI 3 "immediate_operand" "i")
3321                      (match_operand:SI 4 "immediate_operand" "i")]
3322                     UNSPEC_VMUL_LANE))]
3323   "TARGET_NEON"
3324 {
3325   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3326   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3327 }
3328   [(set (attr "neon_type")
3329      (if_then_else (match_test "<Is_float_mode>")
3330                    (const_string "neon_fp_vmul_ddd")
3331                    (if_then_else (match_test "<Scalar_mul_8_16>")
3332                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3333                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3334 )
3335
3336 (define_insn "neon_vmul_lane<mode>"
3337   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3338         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3339                      (match_operand:<V_HALF> 2 "s_register_operand"
3340                                              "<scalar_mul_constraint>")
3341                      (match_operand:SI 3 "immediate_operand" "i")
3342                      (match_operand:SI 4 "immediate_operand" "i")]
3343                     UNSPEC_VMUL_LANE))]
3344   "TARGET_NEON"
3345 {
3346   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3347   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3348 }
3349   [(set (attr "neon_type")
3350      (if_then_else (match_test "<Is_float_mode>")
3351                    (const_string "neon_fp_vmul_qqd")
3352                    (if_then_else (match_test "<Scalar_mul_8_16>")
3353                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3354                                  (const_string "neon_mul_qqd_32_scalar"))))]
3355 )
3356
3357 (define_insn "neon_vmull_lane<mode>"
3358   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3359         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3360                            (match_operand:VMDI 2 "s_register_operand"
3361                                                "<scalar_mul_constraint>")
3362                            (match_operand:SI 3 "immediate_operand" "i")
3363                            (match_operand:SI 4 "immediate_operand" "i")]
3364                           UNSPEC_VMULL_LANE))]
3365   "TARGET_NEON"
3366 {
3367   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3368   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3369 }
3370   [(set (attr "neon_type")
3371      (if_then_else (match_test "<Scalar_mul_8_16>")
3372                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3373                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3374 )
3375
3376 (define_insn "neon_vqdmull_lane<mode>"
3377   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3378         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3379                            (match_operand:VMDI 2 "s_register_operand"
3380                                                "<scalar_mul_constraint>")
3381                            (match_operand:SI 3 "immediate_operand" "i")
3382                            (match_operand:SI 4 "immediate_operand" "i")]
3383                           UNSPEC_VQDMULL_LANE))]
3384   "TARGET_NEON"
3385 {
3386   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3387   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3388 }
3389   [(set (attr "neon_type")
3390      (if_then_else (match_test "<Scalar_mul_8_16>")
3391                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3392                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3393 )
3394
3395 (define_insn "neon_vqdmulh_lane<mode>"
3396   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3397         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3398                       (match_operand:<V_HALF> 2 "s_register_operand"
3399                                               "<scalar_mul_constraint>")
3400                       (match_operand:SI 3 "immediate_operand" "i")
3401                       (match_operand:SI 4 "immediate_operand" "i")]
3402                       UNSPEC_VQDMULH_LANE))]
3403   "TARGET_NEON"
3404 {
3405   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3406   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3407 }
3408   [(set (attr "neon_type")
3409      (if_then_else (match_test "<Scalar_mul_8_16>")
3410                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3411                    (const_string "neon_mul_qqd_32_scalar")))]
3412 )
3413
3414 (define_insn "neon_vqdmulh_lane<mode>"
3415   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3416         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3417                       (match_operand:VMDI 2 "s_register_operand"
3418                                           "<scalar_mul_constraint>")
3419                       (match_operand:SI 3 "immediate_operand" "i")
3420                       (match_operand:SI 4 "immediate_operand" "i")]
3421                       UNSPEC_VQDMULH_LANE))]
3422   "TARGET_NEON"
3423 {
3424   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3425   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3426 }
3427   [(set (attr "neon_type")
3428      (if_then_else (match_test "<Scalar_mul_8_16>")
3429                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3430                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3431 )
3432
3433 (define_insn "neon_vmla_lane<mode>"
3434   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3435         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3436                      (match_operand:VMD 2 "s_register_operand" "w")
3437                      (match_operand:VMD 3 "s_register_operand"
3438                                         "<scalar_mul_constraint>")
3439                      (match_operand:SI 4 "immediate_operand" "i")
3440                      (match_operand:SI 5 "immediate_operand" "i")]
3441                      UNSPEC_VMLA_LANE))]
3442   "TARGET_NEON"
3443 {
3444   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3445   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3446 }
3447   [(set (attr "neon_type")
3448      (if_then_else (match_test "<Is_float_mode>")
3449                    (const_string "neon_fp_vmla_ddd_scalar")
3450                    (if_then_else (match_test "<Scalar_mul_8_16>")
3451                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3452                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3453 )
3454
3455 (define_insn "neon_vmla_lane<mode>"
3456   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3457         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3458                      (match_operand:VMQ 2 "s_register_operand" "w")
3459                      (match_operand:<V_HALF> 3 "s_register_operand"
3460                                              "<scalar_mul_constraint>")
3461                      (match_operand:SI 4 "immediate_operand" "i")
3462                      (match_operand:SI 5 "immediate_operand" "i")]
3463                      UNSPEC_VMLA_LANE))]
3464   "TARGET_NEON"
3465 {
3466   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3467   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3468 }
3469   [(set (attr "neon_type")
3470      (if_then_else (match_test "<Is_float_mode>")
3471                    (const_string "neon_fp_vmla_qqq_scalar")
3472                    (if_then_else (match_test "<Scalar_mul_8_16>")
3473                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3474                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3475 )
3476
3477 (define_insn "neon_vmlal_lane<mode>"
3478   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3479         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3480                            (match_operand:VMDI 2 "s_register_operand" "w")
3481                            (match_operand:VMDI 3 "s_register_operand"
3482                                                "<scalar_mul_constraint>")
3483                            (match_operand:SI 4 "immediate_operand" "i")
3484                            (match_operand:SI 5 "immediate_operand" "i")]
3485                           UNSPEC_VMLAL_LANE))]
3486   "TARGET_NEON"
3487 {
3488   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3489   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3490 }
3491   [(set (attr "neon_type")
3492      (if_then_else (match_test "<Scalar_mul_8_16>")
3493                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3494                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3495 )
3496
3497 (define_insn "neon_vqdmlal_lane<mode>"
3498   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3499         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3500                            (match_operand:VMDI 2 "s_register_operand" "w")
3501                            (match_operand:VMDI 3 "s_register_operand"
3502                                                "<scalar_mul_constraint>")
3503                            (match_operand:SI 4 "immediate_operand" "i")
3504                            (match_operand:SI 5 "immediate_operand" "i")]
3505                           UNSPEC_VQDMLAL_LANE))]
3506   "TARGET_NEON"
3507 {
3508   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3509   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3510 }
3511   [(set (attr "neon_type")
3512      (if_then_else (match_test "<Scalar_mul_8_16>")
3513                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3514                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3515 )
3516
3517 (define_insn "neon_vmls_lane<mode>"
3518   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3519         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3520                      (match_operand:VMD 2 "s_register_operand" "w")
3521                      (match_operand:VMD 3 "s_register_operand"
3522                                         "<scalar_mul_constraint>")
3523                      (match_operand:SI 4 "immediate_operand" "i")
3524                      (match_operand:SI 5 "immediate_operand" "i")]
3525                     UNSPEC_VMLS_LANE))]
3526   "TARGET_NEON"
3527 {
3528   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3529   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3530 }
3531   [(set (attr "neon_type")
3532      (if_then_else (match_test "<Is_float_mode>")
3533                    (const_string "neon_fp_vmla_ddd_scalar")
3534                    (if_then_else (match_test "<Scalar_mul_8_16>")
3535                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3536                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3537 )
3538
3539 (define_insn "neon_vmls_lane<mode>"
3540   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3541         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3542                      (match_operand:VMQ 2 "s_register_operand" "w")
3543                      (match_operand:<V_HALF> 3 "s_register_operand"
3544                                              "<scalar_mul_constraint>")
3545                      (match_operand:SI 4 "immediate_operand" "i")
3546                      (match_operand:SI 5 "immediate_operand" "i")]
3547                     UNSPEC_VMLS_LANE))]
3548   "TARGET_NEON"
3549 {
3550   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3551   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3552 }
3553   [(set (attr "neon_type")
3554      (if_then_else (match_test "<Is_float_mode>")
3555                    (const_string "neon_fp_vmla_qqq_scalar")
3556                    (if_then_else (match_test "<Scalar_mul_8_16>")
3557                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3558                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3559 )
3560
3561 (define_insn "neon_vmlsl_lane<mode>"
3562   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3563         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3564                            (match_operand:VMDI 2 "s_register_operand" "w")
3565                            (match_operand:VMDI 3 "s_register_operand"
3566                                                "<scalar_mul_constraint>")
3567                            (match_operand:SI 4 "immediate_operand" "i")
3568                            (match_operand:SI 5 "immediate_operand" "i")]
3569                           UNSPEC_VMLSL_LANE))]
3570   "TARGET_NEON"
3571 {
3572   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3573   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3574 }
3575   [(set (attr "neon_type")
3576      (if_then_else (match_test "<Scalar_mul_8_16>")
3577                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3578                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3579 )
3580
3581 (define_insn "neon_vqdmlsl_lane<mode>"
3582   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3583         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3584                            (match_operand:VMDI 2 "s_register_operand" "w")
3585                            (match_operand:VMDI 3 "s_register_operand"
3586                                                "<scalar_mul_constraint>")
3587                            (match_operand:SI 4 "immediate_operand" "i")
3588                            (match_operand:SI 5 "immediate_operand" "i")]
3589                           UNSPEC_VQDMLSL_LANE))]
3590   "TARGET_NEON"
3591 {
3592   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3593   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3594 }
3595   [(set (attr "neon_type")
3596      (if_then_else (match_test "<Scalar_mul_8_16>")
3597                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3598                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3599 )
3600
3601 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3602 ; core register into a temp register, then use a scalar taken from that. This
3603 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3604 ; or extracted from another vector. The latter case it's currently better to
3605 ; use the "_lane" variant, and the former case can probably be implemented
3606 ; using vld1_lane, but that hasn't been done yet.
3607
3608 (define_expand "neon_vmul_n<mode>"
3609   [(match_operand:VMD 0 "s_register_operand" "")
3610    (match_operand:VMD 1 "s_register_operand" "")
3611    (match_operand:<V_elem> 2 "s_register_operand" "")
3612    (match_operand:SI 3 "immediate_operand" "")]
3613   "TARGET_NEON"
3614 {
3615   rtx tmp = gen_reg_rtx (<MODE>mode);
3616   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3617   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3618                                        const0_rtx, const0_rtx));
3619   DONE;
3620 })
3621
3622 (define_expand "neon_vmul_n<mode>"
3623   [(match_operand:VMQ 0 "s_register_operand" "")
3624    (match_operand:VMQ 1 "s_register_operand" "")
3625    (match_operand:<V_elem> 2 "s_register_operand" "")
3626    (match_operand:SI 3 "immediate_operand" "")]
3627   "TARGET_NEON"
3628 {
3629   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3630   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3631   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3632                                        const0_rtx, const0_rtx));
3633   DONE;
3634 })
3635
3636 (define_expand "neon_vmull_n<mode>"
3637   [(match_operand:<V_widen> 0 "s_register_operand" "")
3638    (match_operand:VMDI 1 "s_register_operand" "")
3639    (match_operand:<V_elem> 2 "s_register_operand" "")
3640    (match_operand:SI 3 "immediate_operand" "")]
3641   "TARGET_NEON"
3642 {
3643   rtx tmp = gen_reg_rtx (<MODE>mode);
3644   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3645   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3646                                         const0_rtx, operands[3]));
3647   DONE;
3648 })
3649
3650 (define_expand "neon_vqdmull_n<mode>"
3651   [(match_operand:<V_widen> 0 "s_register_operand" "")
3652    (match_operand:VMDI 1 "s_register_operand" "")
3653    (match_operand:<V_elem> 2 "s_register_operand" "")
3654    (match_operand:SI 3 "immediate_operand" "")]
3655   "TARGET_NEON"
3656 {
3657   rtx tmp = gen_reg_rtx (<MODE>mode);
3658   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3659   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3660                                           const0_rtx, const0_rtx));
3661   DONE;
3662 })
3663
3664 (define_expand "neon_vqdmulh_n<mode>"
3665   [(match_operand:VMDI 0 "s_register_operand" "")
3666    (match_operand:VMDI 1 "s_register_operand" "")
3667    (match_operand:<V_elem> 2 "s_register_operand" "")
3668    (match_operand:SI 3 "immediate_operand" "")]
3669   "TARGET_NEON"
3670 {
3671   rtx tmp = gen_reg_rtx (<MODE>mode);
3672   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3673   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3674                                           const0_rtx, operands[3]));
3675   DONE;
3676 })
3677
3678 (define_expand "neon_vqdmulh_n<mode>"
3679   [(match_operand:VMQI 0 "s_register_operand" "")
3680    (match_operand:VMQI 1 "s_register_operand" "")
3681    (match_operand:<V_elem> 2 "s_register_operand" "")
3682    (match_operand:SI 3 "immediate_operand" "")]
3683   "TARGET_NEON"
3684 {
3685   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3686   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3687   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3688                                           const0_rtx, operands[3]));
3689   DONE;
3690 })
3691
3692 (define_expand "neon_vmla_n<mode>"
3693   [(match_operand:VMD 0 "s_register_operand" "")
3694    (match_operand:VMD 1 "s_register_operand" "")
3695    (match_operand:VMD 2 "s_register_operand" "")
3696    (match_operand:<V_elem> 3 "s_register_operand" "")
3697    (match_operand:SI 4 "immediate_operand" "")]
3698   "TARGET_NEON"
3699 {
3700   rtx tmp = gen_reg_rtx (<MODE>mode);
3701   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3702   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3703                                        tmp, const0_rtx, operands[4]));
3704   DONE;
3705 })
3706
3707 (define_expand "neon_vmla_n<mode>"
3708   [(match_operand:VMQ 0 "s_register_operand" "")
3709    (match_operand:VMQ 1 "s_register_operand" "")
3710    (match_operand:VMQ 2 "s_register_operand" "")
3711    (match_operand:<V_elem> 3 "s_register_operand" "")
3712    (match_operand:SI 4 "immediate_operand" "")]
3713   "TARGET_NEON"
3714 {
3715   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3716   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3717   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3718                                        tmp, const0_rtx, operands[4]));
3719   DONE;
3720 })
3721
3722 (define_expand "neon_vmlal_n<mode>"
3723   [(match_operand:<V_widen> 0 "s_register_operand" "")
3724    (match_operand:<V_widen> 1 "s_register_operand" "")
3725    (match_operand:VMDI 2 "s_register_operand" "")
3726    (match_operand:<V_elem> 3 "s_register_operand" "")
3727    (match_operand:SI 4 "immediate_operand" "")]
3728   "TARGET_NEON"
3729 {
3730   rtx tmp = gen_reg_rtx (<MODE>mode);
3731   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3732   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3733                                         tmp, const0_rtx, operands[4]));
3734   DONE;
3735 })
3736
3737 (define_expand "neon_vqdmlal_n<mode>"
3738   [(match_operand:<V_widen> 0 "s_register_operand" "")
3739    (match_operand:<V_widen> 1 "s_register_operand" "")
3740    (match_operand:VMDI 2 "s_register_operand" "")
3741    (match_operand:<V_elem> 3 "s_register_operand" "")
3742    (match_operand:SI 4 "immediate_operand" "")]
3743   "TARGET_NEON"
3744 {
3745   rtx tmp = gen_reg_rtx (<MODE>mode);
3746   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3747   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3748                                           tmp, const0_rtx, operands[4]));
3749   DONE;
3750 })
3751
3752 (define_expand "neon_vmls_n<mode>"
3753   [(match_operand:VMD 0 "s_register_operand" "")
3754    (match_operand:VMD 1 "s_register_operand" "")
3755    (match_operand:VMD 2 "s_register_operand" "")
3756    (match_operand:<V_elem> 3 "s_register_operand" "")
3757    (match_operand:SI 4 "immediate_operand" "")]
3758   "TARGET_NEON"
3759 {
3760   rtx tmp = gen_reg_rtx (<MODE>mode);
3761   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3762   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3763                                        tmp, const0_rtx, operands[4]));
3764   DONE;
3765 })
3766
3767 (define_expand "neon_vmls_n<mode>"
3768   [(match_operand:VMQ 0 "s_register_operand" "")
3769    (match_operand:VMQ 1 "s_register_operand" "")
3770    (match_operand:VMQ 2 "s_register_operand" "")
3771    (match_operand:<V_elem> 3 "s_register_operand" "")
3772    (match_operand:SI 4 "immediate_operand" "")]
3773   "TARGET_NEON"
3774 {
3775   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3776   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3777   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3778                                        tmp, const0_rtx, operands[4]));
3779   DONE;
3780 })
3781
3782 (define_expand "neon_vmlsl_n<mode>"
3783   [(match_operand:<V_widen> 0 "s_register_operand" "")
3784    (match_operand:<V_widen> 1 "s_register_operand" "")
3785    (match_operand:VMDI 2 "s_register_operand" "")
3786    (match_operand:<V_elem> 3 "s_register_operand" "")
3787    (match_operand:SI 4 "immediate_operand" "")]
3788   "TARGET_NEON"
3789 {
3790   rtx tmp = gen_reg_rtx (<MODE>mode);
3791   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3792   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3793                                         tmp, const0_rtx, operands[4]));
3794   DONE;
3795 })
3796
3797 (define_expand "neon_vqdmlsl_n<mode>"
3798   [(match_operand:<V_widen> 0 "s_register_operand" "")
3799    (match_operand:<V_widen> 1 "s_register_operand" "")
3800    (match_operand:VMDI 2 "s_register_operand" "")
3801    (match_operand:<V_elem> 3 "s_register_operand" "")
3802    (match_operand:SI 4 "immediate_operand" "")]
3803   "TARGET_NEON"
3804 {
3805   rtx tmp = gen_reg_rtx (<MODE>mode);
3806   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3807   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3808                                           tmp, const0_rtx, operands[4]));
3809   DONE;
3810 })
3811
3812 (define_insn "neon_vext<mode>"
3813   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3814         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3815                       (match_operand:VDQX 2 "s_register_operand" "w")
3816                       (match_operand:SI 3 "immediate_operand" "i")]
3817                      UNSPEC_VEXT))]
3818   "TARGET_NEON"
3819 {
3820   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3821   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3822 }
3823   [(set (attr "neon_type")
3824       (if_then_else (match_test "<Is_d_reg>")
3825                     (const_string "neon_bp_simple")
3826                     (const_string "neon_bp_2cycle")))]
3827 )
3828
3829 (define_insn "neon_vrev64<mode>"
3830   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3831         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3832                      (match_operand:SI 2 "immediate_operand" "i")]
3833                     UNSPEC_VREV64))]
3834   "TARGET_NEON"
3835   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3836   [(set_attr "neon_type" "neon_bp_simple")]
3837 )
3838
3839 (define_insn "neon_vrev32<mode>"
3840   [(set (match_operand:VX 0 "s_register_operand" "=w")
3841         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3842                     (match_operand:SI 2 "immediate_operand" "i")]
3843                    UNSPEC_VREV32))]
3844   "TARGET_NEON"
3845   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3846   [(set_attr "neon_type" "neon_bp_simple")]
3847 )
3848
3849 (define_insn "neon_vrev16<mode>"
3850   [(set (match_operand:VE 0 "s_register_operand" "=w")
3851         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3852                     (match_operand:SI 2 "immediate_operand" "i")]
3853                    UNSPEC_VREV16))]
3854   "TARGET_NEON"
3855   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3856   [(set_attr "neon_type" "neon_bp_simple")]
3857 )
3858
3859 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3860 ; allocation. For an intrinsic of form:
3861 ;   rD = vbsl_* (rS, rN, rM)
3862 ; We can use any of:
3863 ;   vbsl rS, rN, rM  (if D = S)
3864 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3865 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3866
3867 (define_insn "neon_vbsl<mode>_internal"
3868   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3869         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3870                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3871                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3872                      UNSPEC_VBSL))]
3873   "TARGET_NEON"
3874   "@
3875   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3876   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3877   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3878   [(set_attr "neon_type" "neon_int_1")]
3879 )
3880
3881 (define_expand "neon_vbsl<mode>"
3882   [(set (match_operand:VDQX 0 "s_register_operand" "")
3883         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3884                       (match_operand:VDQX 2 "s_register_operand" "")
3885                       (match_operand:VDQX 3 "s_register_operand" "")]
3886                      UNSPEC_VBSL))]
3887   "TARGET_NEON"
3888 {
3889   /* We can't alias operands together if they have different modes.  */
3890   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3891 })
3892
3893 (define_insn "neon_vshl<mode>"
3894   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3895         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3896                        (match_operand:VDQIX 2 "s_register_operand" "w")
3897                        (match_operand:SI 3 "immediate_operand" "i")]
3898                       UNSPEC_VSHL))]
3899   "TARGET_NEON"
3900   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3901   [(set (attr "neon_type")
3902       (if_then_else (match_test "<Is_d_reg>")
3903                     (const_string "neon_vshl_ddd")
3904                     (const_string "neon_shift_3")))]
3905 )
3906
3907 (define_insn "neon_vqshl<mode>"
3908   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3909         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3910                        (match_operand:VDQIX 2 "s_register_operand" "w")
3911                        (match_operand:SI 3 "immediate_operand" "i")]
3912                       UNSPEC_VQSHL))]
3913   "TARGET_NEON"
3914   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3915   [(set (attr "neon_type")
3916       (if_then_else (match_test "<Is_d_reg>")
3917                     (const_string "neon_shift_2")
3918                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3919 )
3920
3921 (define_insn "neon_vshr_n<mode>"
3922   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3923         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3924                        (match_operand:SI 2 "immediate_operand" "i")
3925                        (match_operand:SI 3 "immediate_operand" "i")]
3926                       UNSPEC_VSHR_N))]
3927   "TARGET_NEON"
3928 {
3929   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3930   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3931 }
3932   [(set_attr "neon_type" "neon_shift_1")]
3933 )
3934
3935 (define_insn "neon_vshrn_n<mode>"
3936   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3937         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3938                             (match_operand:SI 2 "immediate_operand" "i")
3939                             (match_operand:SI 3 "immediate_operand" "i")]
3940                            UNSPEC_VSHRN_N))]
3941   "TARGET_NEON"
3942 {
3943   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3944   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3945 }
3946   [(set_attr "neon_type" "neon_shift_1")]
3947 )
3948
3949 (define_insn "neon_vqshrn_n<mode>"
3950   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3951         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3952                             (match_operand:SI 2 "immediate_operand" "i")
3953                             (match_operand:SI 3 "immediate_operand" "i")]
3954                            UNSPEC_VQSHRN_N))]
3955   "TARGET_NEON"
3956 {
3957   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3958   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3959 }
3960   [(set_attr "neon_type" "neon_shift_2")]
3961 )
3962
3963 (define_insn "neon_vqshrun_n<mode>"
3964   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3965         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3966                             (match_operand:SI 2 "immediate_operand" "i")
3967                             (match_operand:SI 3 "immediate_operand" "i")]
3968                            UNSPEC_VQSHRUN_N))]
3969   "TARGET_NEON"
3970 {
3971   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3972   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3973 }
3974   [(set_attr "neon_type" "neon_shift_2")]
3975 )
3976
3977 (define_insn "neon_vshl_n<mode>"
3978   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3979         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3980                        (match_operand:SI 2 "immediate_operand" "i")
3981                        (match_operand:SI 3 "immediate_operand" "i")]
3982                       UNSPEC_VSHL_N))]
3983   "TARGET_NEON"
3984 {
3985   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3986   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3987 }
3988   [(set_attr "neon_type" "neon_shift_1")]
3989 )
3990
3991 (define_insn "neon_vqshl_n<mode>"
3992   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3993         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3994                        (match_operand:SI 2 "immediate_operand" "i")
3995                        (match_operand:SI 3 "immediate_operand" "i")]
3996                       UNSPEC_VQSHL_N))]
3997   "TARGET_NEON"
3998 {
3999   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4000   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4001 }
4002   [(set_attr "neon_type" "neon_shift_2")]
4003 )
4004
4005 (define_insn "neon_vqshlu_n<mode>"
4006   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4007         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4008                        (match_operand:SI 2 "immediate_operand" "i")
4009                        (match_operand:SI 3 "immediate_operand" "i")]
4010                       UNSPEC_VQSHLU_N))]
4011   "TARGET_NEON"
4012 {
4013   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4014   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4015 }
4016   [(set_attr "neon_type" "neon_shift_2")]
4017 )
4018
4019 (define_insn "neon_vshll_n<mode>"
4020   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
4021         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
4022                            (match_operand:SI 2 "immediate_operand" "i")
4023                            (match_operand:SI 3 "immediate_operand" "i")]
4024                           UNSPEC_VSHLL_N))]
4025   "TARGET_NEON"
4026 {
4027   /* The boundaries are: 0 < imm <= size.  */
4028   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
4029   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
4030 }
4031   [(set_attr "neon_type" "neon_shift_1")]
4032 )
4033
4034 (define_insn "neon_vsra_n<mode>"
4035   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4036         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4037                        (match_operand:VDQIX 2 "s_register_operand" "w")
4038                        (match_operand:SI 3 "immediate_operand" "i")
4039                        (match_operand:SI 4 "immediate_operand" "i")]
4040                       UNSPEC_VSRA_N))]
4041   "TARGET_NEON"
4042 {
4043   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4044   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4045 }
4046   [(set_attr "neon_type" "neon_vsra_vrsra")]
4047 )
4048
4049 (define_insn "neon_vsri_n<mode>"
4050   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4051         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4052                        (match_operand:VDQIX 2 "s_register_operand" "w")
4053                        (match_operand:SI 3 "immediate_operand" "i")]
4054                       UNSPEC_VSRI))]
4055   "TARGET_NEON"
4056 {
4057   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4058   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4059 }
4060   [(set (attr "neon_type")
4061       (if_then_else (match_test "<Is_d_reg>")
4062                     (const_string "neon_shift_1")
4063                     (const_string "neon_shift_3")))]
4064 )
4065
4066 (define_insn "neon_vsli_n<mode>"
4067   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4068         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4069                        (match_operand:VDQIX 2 "s_register_operand" "w")
4070                        (match_operand:SI 3 "immediate_operand" "i")]
4071                       UNSPEC_VSLI))]
4072   "TARGET_NEON"
4073 {
4074   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
4075   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4076 }
4077   [(set (attr "neon_type")
4078       (if_then_else (match_test "<Is_d_reg>")
4079                     (const_string "neon_shift_1")
4080                     (const_string "neon_shift_3")))]
4081 )
4082
4083 (define_insn "neon_vtbl1v8qi"
4084   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4085         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
4086                       (match_operand:V8QI 2 "s_register_operand" "w")]
4087                      UNSPEC_VTBL))]
4088   "TARGET_NEON"
4089   "vtbl.8\t%P0, {%P1}, %P2"
4090   [(set_attr "neon_type" "neon_bp_2cycle")]
4091 )
4092
4093 (define_insn "neon_vtbl2v8qi"
4094   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4095         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
4096                       (match_operand:V8QI 2 "s_register_operand" "w")]
4097                      UNSPEC_VTBL))]
4098   "TARGET_NEON"
4099 {
4100   rtx ops[4];
4101   int tabbase = REGNO (operands[1]);
4102
4103   ops[0] = operands[0];
4104   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4105   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4106   ops[3] = operands[2];
4107   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
4108
4109   return "";
4110 }
4111   [(set_attr "neon_type" "neon_bp_2cycle")]
4112 )
4113
4114 (define_insn "neon_vtbl3v8qi"
4115   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4116         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
4117                       (match_operand:V8QI 2 "s_register_operand" "w")]
4118                      UNSPEC_VTBL))]
4119   "TARGET_NEON"
4120 {
4121   rtx ops[5];
4122   int tabbase = REGNO (operands[1]);
4123
4124   ops[0] = operands[0];
4125   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4126   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4127   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4128   ops[4] = operands[2];
4129   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4130
4131   return "";
4132 }
4133   [(set_attr "neon_type" "neon_bp_3cycle")]
4134 )
4135
4136 (define_insn "neon_vtbl4v8qi"
4137   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4138         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
4139                       (match_operand:V8QI 2 "s_register_operand" "w")]
4140                      UNSPEC_VTBL))]
4141   "TARGET_NEON"
4142 {
4143   rtx ops[6];
4144   int tabbase = REGNO (operands[1]);
4145
4146   ops[0] = operands[0];
4147   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4148   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4149   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4150   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4151   ops[5] = operands[2];
4152   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4153
4154   return "";
4155 }
4156   [(set_attr "neon_type" "neon_bp_3cycle")]
4157 )
4158
4159 ;; These three are used by the vec_perm infrastructure for V16QImode.
4160 (define_insn_and_split "neon_vtbl1v16qi"
4161   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4162         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
4163                        (match_operand:V16QI 2 "s_register_operand" "w")]
4164                       UNSPEC_VTBL))]
4165   "TARGET_NEON"
4166   "#"
4167   "&& reload_completed"
4168   [(const_int 0)]
4169 {
4170   rtx op0, op1, op2, part0, part2;
4171   unsigned ofs;
4172
4173   op0 = operands[0];
4174   op1 = gen_lowpart (TImode, operands[1]);
4175   op2 = operands[2];
4176
4177   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4178   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4179   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4180   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4181
4182   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4183   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4184   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4185   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4186   DONE;
4187 })
4188
4189 (define_insn_and_split "neon_vtbl2v16qi"
4190   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4191         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4192                        (match_operand:V16QI 2 "s_register_operand" "w")]
4193                       UNSPEC_VTBL))]
4194   "TARGET_NEON"
4195   "#"
4196   "&& reload_completed"
4197   [(const_int 0)]
4198 {
4199   rtx op0, op1, op2, part0, part2;
4200   unsigned ofs;
4201
4202   op0 = operands[0];
4203   op1 = operands[1];
4204   op2 = operands[2];
4205
4206   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4207   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4208   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4209   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4210
4211   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4212   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4213   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4214   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4215   DONE;
4216 })
4217
4218 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4219 ;; handle quad-word input modes, producing octa-word output modes.  But
4220 ;; that requires us to add support for octa-word vector modes in moves.
4221 ;; That seems overkill for this one use in vec_perm.
4222 (define_insn_and_split "neon_vcombinev16qi"
4223   [(set (match_operand:OI 0 "s_register_operand" "=w")
4224         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4225                     (match_operand:V16QI 2 "s_register_operand" "w")]
4226                    UNSPEC_VCONCAT))]
4227   "TARGET_NEON"
4228   "#"
4229   "&& reload_completed"
4230   [(const_int 0)]
4231 {
4232   neon_split_vcombine (operands);
4233   DONE;
4234 })
4235
4236 (define_insn "neon_vtbx1v8qi"
4237   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4238         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4239                       (match_operand:V8QI 2 "s_register_operand" "w")
4240                       (match_operand:V8QI 3 "s_register_operand" "w")]
4241                      UNSPEC_VTBX))]
4242   "TARGET_NEON"
4243   "vtbx.8\t%P0, {%P2}, %P3"
4244   [(set_attr "neon_type" "neon_bp_2cycle")]
4245 )
4246
4247 (define_insn "neon_vtbx2v8qi"
4248   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4249         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4250                       (match_operand:TI 2 "s_register_operand" "w")
4251                       (match_operand:V8QI 3 "s_register_operand" "w")]
4252                      UNSPEC_VTBX))]
4253   "TARGET_NEON"
4254 {
4255   rtx ops[4];
4256   int tabbase = REGNO (operands[2]);
4257
4258   ops[0] = operands[0];
4259   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4260   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4261   ops[3] = operands[3];
4262   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4263
4264   return "";
4265 }
4266   [(set_attr "neon_type" "neon_bp_2cycle")]
4267 )
4268
4269 (define_insn "neon_vtbx3v8qi"
4270   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4271         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4272                       (match_operand:EI 2 "s_register_operand" "w")
4273                       (match_operand:V8QI 3 "s_register_operand" "w")]
4274                      UNSPEC_VTBX))]
4275   "TARGET_NEON"
4276 {
4277   rtx ops[5];
4278   int tabbase = REGNO (operands[2]);
4279
4280   ops[0] = operands[0];
4281   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4282   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4283   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4284   ops[4] = operands[3];
4285   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4286
4287   return "";
4288 }
4289   [(set_attr "neon_type" "neon_bp_3cycle")]
4290 )
4291
4292 (define_insn "neon_vtbx4v8qi"
4293   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4294         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4295                       (match_operand:OI 2 "s_register_operand" "w")
4296                       (match_operand:V8QI 3 "s_register_operand" "w")]
4297                      UNSPEC_VTBX))]
4298   "TARGET_NEON"
4299 {
4300   rtx ops[6];
4301   int tabbase = REGNO (operands[2]);
4302
4303   ops[0] = operands[0];
4304   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4305   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4306   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4307   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4308   ops[5] = operands[3];
4309   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4310
4311   return "";
4312 }
4313   [(set_attr "neon_type" "neon_bp_3cycle")]
4314 )
4315
4316 (define_expand "neon_vtrn<mode>_internal"
4317   [(parallel
4318     [(set (match_operand:VDQW 0 "s_register_operand" "")
4319           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4320                         (match_operand:VDQW 2 "s_register_operand" "")]
4321            UNSPEC_VTRN1))
4322      (set (match_operand:VDQW 3 "s_register_operand" "")
4323           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VTRN2))])]
4324   "TARGET_NEON"
4325   ""
4326 )
4327
4328 ;; Note: Different operand numbering to handle tied registers correctly.
4329 (define_insn "*neon_vtrn<mode>_insn"
4330   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4331         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4332                       (match_operand:VDQW 3 "s_register_operand" "2")]
4333                      UNSPEC_VTRN1))
4334    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4335          (unspec:VDQW [(match_dup 1) (match_dup 3)]
4336                      UNSPEC_VTRN2))]
4337   "TARGET_NEON"
4338   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4339   [(set (attr "neon_type")
4340       (if_then_else (match_test "<Is_d_reg>")
4341                     (const_string "neon_bp_simple")
4342                     (const_string "neon_bp_3cycle")))]
4343 )
4344
4345 (define_expand "neon_vtrn<mode>"
4346   [(match_operand:SI 0 "s_register_operand" "r")
4347    (match_operand:VDQW 1 "s_register_operand" "w")
4348    (match_operand:VDQW 2 "s_register_operand" "w")]
4349   "TARGET_NEON"
4350 {
4351   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4352                               operands[0], operands[1], operands[2]);
4353   DONE;
4354 })
4355
4356 (define_expand "neon_vzip<mode>_internal"
4357   [(parallel
4358     [(set (match_operand:VDQW 0 "s_register_operand" "")
4359           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4360                         (match_operand:VDQW 2 "s_register_operand" "")]
4361                        UNSPEC_VZIP1))
4362     (set (match_operand:VDQW 3 "s_register_operand" "")
4363          (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VZIP2))])]
4364   "TARGET_NEON"
4365   ""
4366 )
4367
4368 ;; Note: Different operand numbering to handle tied registers correctly.
4369 (define_insn "*neon_vzip<mode>_insn"
4370   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4371         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4372                       (match_operand:VDQW 3 "s_register_operand" "2")]
4373                      UNSPEC_VZIP1))
4374    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4375         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4376                      UNSPEC_VZIP2))]
4377   "TARGET_NEON"
4378   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4379   [(set (attr "neon_type")
4380       (if_then_else (match_test "<Is_d_reg>")
4381                     (const_string "neon_bp_simple")
4382                     (const_string "neon_bp_3cycle")))]
4383 )
4384
4385 (define_expand "neon_vzip<mode>"
4386   [(match_operand:SI 0 "s_register_operand" "r")
4387    (match_operand:VDQW 1 "s_register_operand" "w")
4388    (match_operand:VDQW 2 "s_register_operand" "w")]
4389   "TARGET_NEON"
4390 {
4391   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4392                               operands[0], operands[1], operands[2]);
4393   DONE;
4394 })
4395
4396 (define_expand "neon_vuzp<mode>_internal"
4397   [(parallel
4398     [(set (match_operand:VDQW 0 "s_register_operand" "")
4399           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4400                         (match_operand:VDQW 2 "s_register_operand" "")]
4401            UNSPEC_VUZP1))
4402      (set (match_operand:VDQW 3 "s_register_operand" "")
4403           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VUZP2))])]
4404   "TARGET_NEON"
4405   ""
4406 )
4407
4408 ;; Note: Different operand numbering to handle tied registers correctly.
4409 (define_insn "*neon_vuzp<mode>_insn"
4410   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4411         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4412                       (match_operand:VDQW 3 "s_register_operand" "2")]
4413                      UNSPEC_VUZP1))
4414    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4415         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4416                      UNSPEC_VUZP2))]
4417   "TARGET_NEON"
4418   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4419   [(set (attr "neon_type")
4420       (if_then_else (match_test "<Is_d_reg>")
4421                     (const_string "neon_bp_simple")
4422                     (const_string "neon_bp_3cycle")))]
4423 )
4424
4425 (define_expand "neon_vuzp<mode>"
4426   [(match_operand:SI 0 "s_register_operand" "r")
4427    (match_operand:VDQW 1 "s_register_operand" "w")
4428    (match_operand:VDQW 2 "s_register_operand" "w")]
4429   "TARGET_NEON"
4430 {
4431   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4432                               operands[0], operands[1], operands[2]);
4433   DONE;
4434 })
4435
4436 (define_expand "neon_vreinterpretv8qi<mode>"
4437   [(match_operand:V8QI 0 "s_register_operand" "")
4438    (match_operand:VDX 1 "s_register_operand" "")]
4439   "TARGET_NEON"
4440 {
4441   neon_reinterpret (operands[0], operands[1]);
4442   DONE;
4443 })
4444
4445 (define_expand "neon_vreinterpretv4hi<mode>"
4446   [(match_operand:V4HI 0 "s_register_operand" "")
4447    (match_operand:VDX 1 "s_register_operand" "")]
4448   "TARGET_NEON"
4449 {
4450   neon_reinterpret (operands[0], operands[1]);
4451   DONE;
4452 })
4453
4454 (define_expand "neon_vreinterpretv2si<mode>"
4455   [(match_operand:V2SI 0 "s_register_operand" "")
4456    (match_operand:VDX 1 "s_register_operand" "")]
4457   "TARGET_NEON"
4458 {
4459   neon_reinterpret (operands[0], operands[1]);
4460   DONE;
4461 })
4462
4463 (define_expand "neon_vreinterpretv2sf<mode>"
4464   [(match_operand:V2SF 0 "s_register_operand" "")
4465    (match_operand:VDX 1 "s_register_operand" "")]
4466   "TARGET_NEON"
4467 {
4468   neon_reinterpret (operands[0], operands[1]);
4469   DONE;
4470 })
4471
4472 (define_expand "neon_vreinterpretdi<mode>"
4473   [(match_operand:DI 0 "s_register_operand" "")
4474    (match_operand:VDX 1 "s_register_operand" "")]
4475   "TARGET_NEON"
4476 {
4477   neon_reinterpret (operands[0], operands[1]);
4478   DONE;
4479 })
4480
4481 (define_expand "neon_vreinterpretv16qi<mode>"
4482   [(match_operand:V16QI 0 "s_register_operand" "")
4483    (match_operand:VQX 1 "s_register_operand" "")]
4484   "TARGET_NEON"
4485 {
4486   neon_reinterpret (operands[0], operands[1]);
4487   DONE;
4488 })
4489
4490 (define_expand "neon_vreinterpretv8hi<mode>"
4491   [(match_operand:V8HI 0 "s_register_operand" "")
4492    (match_operand:VQX 1 "s_register_operand" "")]
4493   "TARGET_NEON"
4494 {
4495   neon_reinterpret (operands[0], operands[1]);
4496   DONE;
4497 })
4498
4499 (define_expand "neon_vreinterpretv4si<mode>"
4500   [(match_operand:V4SI 0 "s_register_operand" "")
4501    (match_operand:VQX 1 "s_register_operand" "")]
4502   "TARGET_NEON"
4503 {
4504   neon_reinterpret (operands[0], operands[1]);
4505   DONE;
4506 })
4507
4508 (define_expand "neon_vreinterpretv4sf<mode>"
4509   [(match_operand:V4SF 0 "s_register_operand" "")
4510    (match_operand:VQX 1 "s_register_operand" "")]
4511   "TARGET_NEON"
4512 {
4513   neon_reinterpret (operands[0], operands[1]);
4514   DONE;
4515 })
4516
4517 (define_expand "neon_vreinterpretv2di<mode>"
4518   [(match_operand:V2DI 0 "s_register_operand" "")
4519    (match_operand:VQX 1 "s_register_operand" "")]
4520   "TARGET_NEON"
4521 {
4522   neon_reinterpret (operands[0], operands[1]);
4523   DONE;
4524 })
4525
4526 (define_expand "vec_load_lanes<mode><mode>"
4527   [(set (match_operand:VDQX 0 "s_register_operand")
4528         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4529                      UNSPEC_VLD1))]
4530   "TARGET_NEON")
4531
4532 (define_insn "neon_vld1<mode>"
4533   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4534         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4535                     UNSPEC_VLD1))]
4536   "TARGET_NEON"
4537   "vld1.<V_sz_elem>\t%h0, %A1"
4538   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4539 )
4540
4541 (define_insn "neon_vld1_lane<mode>"
4542   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4543         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4544                      (match_operand:VDX 2 "s_register_operand" "0")
4545                      (match_operand:SI 3 "immediate_operand" "i")]
4546                     UNSPEC_VLD1_LANE))]
4547   "TARGET_NEON"
4548 {
4549   HOST_WIDE_INT lane = INTVAL (operands[3]);
4550   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4551   if (lane < 0 || lane >= max)
4552     error ("lane out of range");
4553   if (max == 1)
4554     return "vld1.<V_sz_elem>\t%P0, %A1";
4555   else
4556     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4557 }
4558   [(set (attr "neon_type")
4559       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4560                     (const_string "neon_vld1_1_2_regs")
4561                     (const_string "neon_vld1_vld2_lane")))]
4562 )
4563
4564 (define_insn "neon_vld1_lane<mode>"
4565   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4566         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4567                      (match_operand:VQX 2 "s_register_operand" "0")
4568                      (match_operand:SI 3 "immediate_operand" "i")]
4569                     UNSPEC_VLD1_LANE))]
4570   "TARGET_NEON"
4571 {
4572   HOST_WIDE_INT lane = INTVAL (operands[3]);
4573   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4574   int regno = REGNO (operands[0]);
4575   if (lane < 0 || lane >= max)
4576     error ("lane out of range");
4577   else if (lane >= max / 2)
4578     {
4579       lane -= max / 2;
4580       regno += 2;
4581       operands[3] = GEN_INT (lane);
4582     }
4583   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4584   if (max == 2)
4585     return "vld1.<V_sz_elem>\t%P0, %A1";
4586   else
4587     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4588 }
4589   [(set (attr "neon_type")
4590       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4591                     (const_string "neon_vld1_1_2_regs")
4592                     (const_string "neon_vld1_vld2_lane")))]
4593 )
4594
4595 (define_insn "neon_vld1_dup<mode>"
4596   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4597         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4598   "TARGET_NEON"
4599 {
4600   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4601     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4602   else
4603     return "vld1.<V_sz_elem>\t%h0, %A1";
4604 }
4605   [(set (attr "neon_type")
4606       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4607                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4608                     (const_string "neon_vld1_1_2_regs")))]
4609 )
4610
4611 (define_insn "neon_vld1_dup<mode>"
4612   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4613         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4614   "TARGET_NEON"
4615 {
4616   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4617 }
4618   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4619 )
4620
4621 (define_insn_and_split "neon_vld1_dupv2di"
4622    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4623     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4624    "TARGET_NEON"
4625    "#"
4626    "&& reload_completed"
4627    [(const_int 0)]
4628    {
4629     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4630     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4631     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4632     DONE;
4633     }
4634   [(set_attr "length" "8")
4635    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4636 )
4637
4638 (define_expand "vec_store_lanes<mode><mode>"
4639   [(set (match_operand:VDQX 0 "neon_struct_operand")
4640         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4641                      UNSPEC_VST1))]
4642   "TARGET_NEON")
4643
4644 (define_insn "neon_vst1<mode>"
4645   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4646         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4647                      UNSPEC_VST1))]
4648   "TARGET_NEON"
4649   "vst1.<V_sz_elem>\t%h1, %A0"
4650   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4651
4652 (define_insn "neon_vst1_lane<mode>"
4653   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4654         (unspec:<V_elem>
4655           [(match_operand:VDX 1 "s_register_operand" "w")
4656            (match_operand:SI 2 "immediate_operand" "i")]
4657           UNSPEC_VST1_LANE))]
4658   "TARGET_NEON"
4659 {
4660   HOST_WIDE_INT lane = INTVAL (operands[2]);
4661   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4662   if (lane < 0 || lane >= max)
4663     error ("lane out of range");
4664   if (max == 1)
4665     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4666   else
4667     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4668 }
4669   [(set (attr "neon_type")
4670       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4671                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4672                     (const_string "neon_vst1_vst2_lane")))])
4673
4674 (define_insn "neon_vst1_lane<mode>"
4675   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4676         (unspec:<V_elem>
4677           [(match_operand:VQX 1 "s_register_operand" "w")
4678            (match_operand:SI 2 "immediate_operand" "i")]
4679           UNSPEC_VST1_LANE))]
4680   "TARGET_NEON"
4681 {
4682   HOST_WIDE_INT lane = INTVAL (operands[2]);
4683   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4684   int regno = REGNO (operands[1]);
4685   if (lane < 0 || lane >= max)
4686     error ("lane out of range");
4687   else if (lane >= max / 2)
4688     {
4689       lane -= max / 2;
4690       regno += 2;
4691       operands[2] = GEN_INT (lane);
4692     }
4693   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4694   if (max == 2)
4695     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4696   else
4697     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4698 }
4699   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4700 )
4701
4702 (define_expand "vec_load_lanesti<mode>"
4703   [(set (match_operand:TI 0 "s_register_operand")
4704         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4705                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4706                    UNSPEC_VLD2))]
4707   "TARGET_NEON")
4708
4709 (define_insn "neon_vld2<mode>"
4710   [(set (match_operand:TI 0 "s_register_operand" "=w")
4711         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4712                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4713                    UNSPEC_VLD2))]
4714   "TARGET_NEON"
4715 {
4716   if (<V_sz_elem> == 64)
4717     return "vld1.64\t%h0, %A1";
4718   else
4719     return "vld2.<V_sz_elem>\t%h0, %A1";
4720 }
4721   [(set (attr "neon_type")
4722       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4723                     (const_string "neon_vld1_1_2_regs")
4724                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4725 )
4726
4727 (define_expand "vec_load_lanesoi<mode>"
4728   [(set (match_operand:OI 0 "s_register_operand")
4729         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4730                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4731                    UNSPEC_VLD2))]
4732   "TARGET_NEON")
4733
4734 (define_insn "neon_vld2<mode>"
4735   [(set (match_operand:OI 0 "s_register_operand" "=w")
4736         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4737                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4738                    UNSPEC_VLD2))]
4739   "TARGET_NEON"
4740   "vld2.<V_sz_elem>\t%h0, %A1"
4741   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4742
4743 (define_insn "neon_vld2_lane<mode>"
4744   [(set (match_operand:TI 0 "s_register_operand" "=w")
4745         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4746                     (match_operand:TI 2 "s_register_operand" "0")
4747                     (match_operand:SI 3 "immediate_operand" "i")
4748                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4749                    UNSPEC_VLD2_LANE))]
4750   "TARGET_NEON"
4751 {
4752   HOST_WIDE_INT lane = INTVAL (operands[3]);
4753   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4754   int regno = REGNO (operands[0]);
4755   rtx ops[4];
4756   if (lane < 0 || lane >= max)
4757     error ("lane out of range");
4758   ops[0] = gen_rtx_REG (DImode, regno);
4759   ops[1] = gen_rtx_REG (DImode, regno + 2);
4760   ops[2] = operands[1];
4761   ops[3] = operands[3];
4762   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4763   return "";
4764 }
4765   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4766 )
4767
4768 (define_insn "neon_vld2_lane<mode>"
4769   [(set (match_operand:OI 0 "s_register_operand" "=w")
4770         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4771                     (match_operand:OI 2 "s_register_operand" "0")
4772                     (match_operand:SI 3 "immediate_operand" "i")
4773                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4774                    UNSPEC_VLD2_LANE))]
4775   "TARGET_NEON"
4776 {
4777   HOST_WIDE_INT lane = INTVAL (operands[3]);
4778   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4779   int regno = REGNO (operands[0]);
4780   rtx ops[4];
4781   if (lane < 0 || lane >= max)
4782     error ("lane out of range");
4783   else if (lane >= max / 2)
4784     {
4785       lane -= max / 2;
4786       regno += 2;
4787     }
4788   ops[0] = gen_rtx_REG (DImode, regno);
4789   ops[1] = gen_rtx_REG (DImode, regno + 4);
4790   ops[2] = operands[1];
4791   ops[3] = GEN_INT (lane);
4792   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4793   return "";
4794 }
4795   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4796 )
4797
4798 (define_insn "neon_vld2_dup<mode>"
4799   [(set (match_operand:TI 0 "s_register_operand" "=w")
4800         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4801                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4802                    UNSPEC_VLD2_DUP))]
4803   "TARGET_NEON"
4804 {
4805   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4806     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4807   else
4808     return "vld1.<V_sz_elem>\t%h0, %A1";
4809 }
4810   [(set (attr "neon_type")
4811       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4812                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4813                     (const_string "neon_vld1_1_2_regs")))]
4814 )
4815
4816 (define_expand "vec_store_lanesti<mode>"
4817   [(set (match_operand:TI 0 "neon_struct_operand")
4818         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4819                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4820                    UNSPEC_VST2))]
4821   "TARGET_NEON")
4822
4823 (define_insn "neon_vst2<mode>"
4824   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4825         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4826                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4827                    UNSPEC_VST2))]
4828   "TARGET_NEON"
4829 {
4830   if (<V_sz_elem> == 64)
4831     return "vst1.64\t%h1, %A0";
4832   else
4833     return "vst2.<V_sz_elem>\t%h1, %A0";
4834 }
4835   [(set (attr "neon_type")
4836       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4837                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4838                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4839 )
4840
4841 (define_expand "vec_store_lanesoi<mode>"
4842   [(set (match_operand:OI 0 "neon_struct_operand")
4843         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4844                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4845                    UNSPEC_VST2))]
4846   "TARGET_NEON")
4847
4848 (define_insn "neon_vst2<mode>"
4849   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4850         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4851                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4852                    UNSPEC_VST2))]
4853   "TARGET_NEON"
4854   "vst2.<V_sz_elem>\t%h1, %A0"
4855   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4856 )
4857
4858 (define_insn "neon_vst2_lane<mode>"
4859   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4860         (unspec:<V_two_elem>
4861           [(match_operand:TI 1 "s_register_operand" "w")
4862            (match_operand:SI 2 "immediate_operand" "i")
4863            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4864           UNSPEC_VST2_LANE))]
4865   "TARGET_NEON"
4866 {
4867   HOST_WIDE_INT lane = INTVAL (operands[2]);
4868   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4869   int regno = REGNO (operands[1]);
4870   rtx ops[4];
4871   if (lane < 0 || lane >= max)
4872     error ("lane out of range");
4873   ops[0] = operands[0];
4874   ops[1] = gen_rtx_REG (DImode, regno);
4875   ops[2] = gen_rtx_REG (DImode, regno + 2);
4876   ops[3] = operands[2];
4877   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4878   return "";
4879 }
4880   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4881 )
4882
4883 (define_insn "neon_vst2_lane<mode>"
4884   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4885         (unspec:<V_two_elem>
4886            [(match_operand:OI 1 "s_register_operand" "w")
4887             (match_operand:SI 2 "immediate_operand" "i")
4888             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4889            UNSPEC_VST2_LANE))]
4890   "TARGET_NEON"
4891 {
4892   HOST_WIDE_INT lane = INTVAL (operands[2]);
4893   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4894   int regno = REGNO (operands[1]);
4895   rtx ops[4];
4896   if (lane < 0 || lane >= max)
4897     error ("lane out of range");
4898   else if (lane >= max / 2)
4899     {
4900       lane -= max / 2;
4901       regno += 2;
4902     }
4903   ops[0] = operands[0];
4904   ops[1] = gen_rtx_REG (DImode, regno);
4905   ops[2] = gen_rtx_REG (DImode, regno + 4);
4906   ops[3] = GEN_INT (lane);
4907   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4908   return "";
4909 }
4910   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4911 )
4912
4913 (define_expand "vec_load_lanesei<mode>"
4914   [(set (match_operand:EI 0 "s_register_operand")
4915         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4916                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4917                    UNSPEC_VLD3))]
4918   "TARGET_NEON")
4919
4920 (define_insn "neon_vld3<mode>"
4921   [(set (match_operand:EI 0 "s_register_operand" "=w")
4922         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4923                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4924                    UNSPEC_VLD3))]
4925   "TARGET_NEON"
4926 {
4927   if (<V_sz_elem> == 64)
4928     return "vld1.64\t%h0, %A1";
4929   else
4930     return "vld3.<V_sz_elem>\t%h0, %A1";
4931 }
4932   [(set (attr "neon_type")
4933       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4934                     (const_string "neon_vld1_1_2_regs")
4935                     (const_string "neon_vld3_vld4")))]
4936 )
4937
4938 (define_expand "vec_load_lanesci<mode>"
4939   [(match_operand:CI 0 "s_register_operand")
4940    (match_operand:CI 1 "neon_struct_operand")
4941    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4942   "TARGET_NEON"
4943 {
4944   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4945   DONE;
4946 })
4947
4948 (define_expand "neon_vld3<mode>"
4949   [(match_operand:CI 0 "s_register_operand")
4950    (match_operand:CI 1 "neon_struct_operand")
4951    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4952   "TARGET_NEON"
4953 {
4954   rtx mem;
4955
4956   mem = adjust_address (operands[1], EImode, 0);
4957   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4958   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4959   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4960   DONE;
4961 })
4962
4963 (define_insn "neon_vld3qa<mode>"
4964   [(set (match_operand:CI 0 "s_register_operand" "=w")
4965         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4966                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4967                    UNSPEC_VLD3A))]
4968   "TARGET_NEON"
4969 {
4970   int regno = REGNO (operands[0]);
4971   rtx ops[4];
4972   ops[0] = gen_rtx_REG (DImode, regno);
4973   ops[1] = gen_rtx_REG (DImode, regno + 4);
4974   ops[2] = gen_rtx_REG (DImode, regno + 8);
4975   ops[3] = operands[1];
4976   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4977   return "";
4978 }
4979   [(set_attr "neon_type" "neon_vld3_vld4")]
4980 )
4981
4982 (define_insn "neon_vld3qb<mode>"
4983   [(set (match_operand:CI 0 "s_register_operand" "=w")
4984         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4985                     (match_operand:CI 2 "s_register_operand" "0")
4986                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4987                    UNSPEC_VLD3B))]
4988   "TARGET_NEON"
4989 {
4990   int regno = REGNO (operands[0]);
4991   rtx ops[4];
4992   ops[0] = gen_rtx_REG (DImode, regno + 2);
4993   ops[1] = gen_rtx_REG (DImode, regno + 6);
4994   ops[2] = gen_rtx_REG (DImode, regno + 10);
4995   ops[3] = operands[1];
4996   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4997   return "";
4998 }
4999   [(set_attr "neon_type" "neon_vld3_vld4")]
5000 )
5001
5002 (define_insn "neon_vld3_lane<mode>"
5003   [(set (match_operand:EI 0 "s_register_operand" "=w")
5004         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5005                     (match_operand:EI 2 "s_register_operand" "0")
5006                     (match_operand:SI 3 "immediate_operand" "i")
5007                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5008                    UNSPEC_VLD3_LANE))]
5009   "TARGET_NEON"
5010 {
5011   HOST_WIDE_INT lane = INTVAL (operands[3]);
5012   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5013   int regno = REGNO (operands[0]);
5014   rtx ops[5];
5015   if (lane < 0 || lane >= max)
5016     error ("lane out of range");
5017   ops[0] = gen_rtx_REG (DImode, regno);
5018   ops[1] = gen_rtx_REG (DImode, regno + 2);
5019   ops[2] = gen_rtx_REG (DImode, regno + 4);
5020   ops[3] = operands[1];
5021   ops[4] = operands[3];
5022   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5023                    ops);
5024   return "";
5025 }
5026   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5027 )
5028
5029 (define_insn "neon_vld3_lane<mode>"
5030   [(set (match_operand:CI 0 "s_register_operand" "=w")
5031         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5032                     (match_operand:CI 2 "s_register_operand" "0")
5033                     (match_operand:SI 3 "immediate_operand" "i")
5034                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5035                    UNSPEC_VLD3_LANE))]
5036   "TARGET_NEON"
5037 {
5038   HOST_WIDE_INT lane = INTVAL (operands[3]);
5039   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5040   int regno = REGNO (operands[0]);
5041   rtx ops[5];
5042   if (lane < 0 || lane >= max)
5043     error ("lane out of range");
5044   else if (lane >= max / 2)
5045     {
5046       lane -= max / 2;
5047       regno += 2;
5048     }
5049   ops[0] = gen_rtx_REG (DImode, regno);
5050   ops[1] = gen_rtx_REG (DImode, regno + 4);
5051   ops[2] = gen_rtx_REG (DImode, regno + 8);
5052   ops[3] = operands[1];
5053   ops[4] = GEN_INT (lane);
5054   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5055                    ops);
5056   return "";
5057 }
5058   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5059 )
5060
5061 (define_insn "neon_vld3_dup<mode>"
5062   [(set (match_operand:EI 0 "s_register_operand" "=w")
5063         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5064                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5065                    UNSPEC_VLD3_DUP))]
5066   "TARGET_NEON"
5067 {
5068   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5069     {
5070       int regno = REGNO (operands[0]);
5071       rtx ops[4];
5072       ops[0] = gen_rtx_REG (DImode, regno);
5073       ops[1] = gen_rtx_REG (DImode, regno + 2);
5074       ops[2] = gen_rtx_REG (DImode, regno + 4);
5075       ops[3] = operands[1];
5076       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
5077       return "";
5078     }
5079   else
5080     return "vld1.<V_sz_elem>\t%h0, %A1";
5081 }
5082   [(set (attr "neon_type")
5083       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5084                     (const_string "neon_vld3_vld4_all_lanes")
5085                     (const_string "neon_vld1_1_2_regs")))])
5086
5087 (define_expand "vec_store_lanesei<mode>"
5088   [(set (match_operand:EI 0 "neon_struct_operand")
5089         (unspec:EI [(match_operand:EI 1 "s_register_operand")
5090                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5091                    UNSPEC_VST3))]
5092   "TARGET_NEON")
5093
5094 (define_insn "neon_vst3<mode>"
5095   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5096         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
5097                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5098                    UNSPEC_VST3))]
5099   "TARGET_NEON"
5100 {
5101   if (<V_sz_elem> == 64)
5102     return "vst1.64\t%h1, %A0";
5103   else
5104     return "vst3.<V_sz_elem>\t%h1, %A0";
5105 }
5106   [(set (attr "neon_type")
5107       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5108                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5109                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
5110
5111 (define_expand "vec_store_lanesci<mode>"
5112   [(match_operand:CI 0 "neon_struct_operand")
5113    (match_operand:CI 1 "s_register_operand")
5114    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5115   "TARGET_NEON"
5116 {
5117   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
5118   DONE;
5119 })
5120
5121 (define_expand "neon_vst3<mode>"
5122   [(match_operand:CI 0 "neon_struct_operand")
5123    (match_operand:CI 1 "s_register_operand")
5124    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5125   "TARGET_NEON"
5126 {
5127   rtx mem;
5128
5129   mem = adjust_address (operands[0], EImode, 0);
5130   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
5131   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
5132   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
5133   DONE;
5134 })
5135
5136 (define_insn "neon_vst3qa<mode>"
5137   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5138         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5139                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5140                    UNSPEC_VST3A))]
5141   "TARGET_NEON"
5142 {
5143   int regno = REGNO (operands[1]);
5144   rtx ops[4];
5145   ops[0] = operands[0];
5146   ops[1] = gen_rtx_REG (DImode, regno);
5147   ops[2] = gen_rtx_REG (DImode, regno + 4);
5148   ops[3] = gen_rtx_REG (DImode, regno + 8);
5149   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5150   return "";
5151 }
5152   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5153 )
5154
5155 (define_insn "neon_vst3qb<mode>"
5156   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5157         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5158                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5159                    UNSPEC_VST3B))]
5160   "TARGET_NEON"
5161 {
5162   int regno = REGNO (operands[1]);
5163   rtx ops[4];
5164   ops[0] = operands[0];
5165   ops[1] = gen_rtx_REG (DImode, regno + 2);
5166   ops[2] = gen_rtx_REG (DImode, regno + 6);
5167   ops[3] = gen_rtx_REG (DImode, regno + 10);
5168   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5169   return "";
5170 }
5171   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5172 )
5173
5174 (define_insn "neon_vst3_lane<mode>"
5175   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5176         (unspec:<V_three_elem>
5177            [(match_operand:EI 1 "s_register_operand" "w")
5178             (match_operand:SI 2 "immediate_operand" "i")
5179             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5180            UNSPEC_VST3_LANE))]
5181   "TARGET_NEON"
5182 {
5183   HOST_WIDE_INT lane = INTVAL (operands[2]);
5184   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5185   int regno = REGNO (operands[1]);
5186   rtx ops[5];
5187   if (lane < 0 || lane >= max)
5188     error ("lane out of range");
5189   ops[0] = operands[0];
5190   ops[1] = gen_rtx_REG (DImode, regno);
5191   ops[2] = gen_rtx_REG (DImode, regno + 2);
5192   ops[3] = gen_rtx_REG (DImode, regno + 4);
5193   ops[4] = operands[2];
5194   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5195                    ops);
5196   return "";
5197 }
5198   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5199 )
5200
5201 (define_insn "neon_vst3_lane<mode>"
5202   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5203         (unspec:<V_three_elem>
5204            [(match_operand:CI 1 "s_register_operand" "w")
5205             (match_operand:SI 2 "immediate_operand" "i")
5206             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5207            UNSPEC_VST3_LANE))]
5208   "TARGET_NEON"
5209 {
5210   HOST_WIDE_INT lane = INTVAL (operands[2]);
5211   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5212   int regno = REGNO (operands[1]);
5213   rtx ops[5];
5214   if (lane < 0 || lane >= max)
5215     error ("lane out of range");
5216   else if (lane >= max / 2)
5217     {
5218       lane -= max / 2;
5219       regno += 2;
5220     }
5221   ops[0] = operands[0];
5222   ops[1] = gen_rtx_REG (DImode, regno);
5223   ops[2] = gen_rtx_REG (DImode, regno + 4);
5224   ops[3] = gen_rtx_REG (DImode, regno + 8);
5225   ops[4] = GEN_INT (lane);
5226   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5227                    ops);
5228   return "";
5229 }
5230 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5231
5232 (define_expand "vec_load_lanesoi<mode>"
5233   [(set (match_operand:OI 0 "s_register_operand")
5234         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5235                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5236                    UNSPEC_VLD4))]
5237   "TARGET_NEON")
5238
5239 (define_insn "neon_vld4<mode>"
5240   [(set (match_operand:OI 0 "s_register_operand" "=w")
5241         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5242                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5243                    UNSPEC_VLD4))]
5244   "TARGET_NEON"
5245 {
5246   if (<V_sz_elem> == 64)
5247     return "vld1.64\t%h0, %A1";
5248   else
5249     return "vld4.<V_sz_elem>\t%h0, %A1";
5250 }
5251   [(set (attr "neon_type")
5252       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5253                     (const_string "neon_vld1_1_2_regs")
5254                     (const_string "neon_vld3_vld4")))]
5255 )
5256
5257 (define_expand "vec_load_lanesxi<mode>"
5258   [(match_operand:XI 0 "s_register_operand")
5259    (match_operand:XI 1 "neon_struct_operand")
5260    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5261   "TARGET_NEON"
5262 {
5263   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5264   DONE;
5265 })
5266
5267 (define_expand "neon_vld4<mode>"
5268   [(match_operand:XI 0 "s_register_operand")
5269    (match_operand:XI 1 "neon_struct_operand")
5270    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5271   "TARGET_NEON"
5272 {
5273   rtx mem;
5274
5275   mem = adjust_address (operands[1], OImode, 0);
5276   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5277   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5278   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5279   DONE;
5280 })
5281
5282 (define_insn "neon_vld4qa<mode>"
5283   [(set (match_operand:XI 0 "s_register_operand" "=w")
5284         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5285                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5286                    UNSPEC_VLD4A))]
5287   "TARGET_NEON"
5288 {
5289   int regno = REGNO (operands[0]);
5290   rtx ops[5];
5291   ops[0] = gen_rtx_REG (DImode, regno);
5292   ops[1] = gen_rtx_REG (DImode, regno + 4);
5293   ops[2] = gen_rtx_REG (DImode, regno + 8);
5294   ops[3] = gen_rtx_REG (DImode, regno + 12);
5295   ops[4] = operands[1];
5296   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5297   return "";
5298 }
5299   [(set_attr "neon_type" "neon_vld3_vld4")]
5300 )
5301
5302 (define_insn "neon_vld4qb<mode>"
5303   [(set (match_operand:XI 0 "s_register_operand" "=w")
5304         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5305                     (match_operand:XI 2 "s_register_operand" "0")
5306                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5307                    UNSPEC_VLD4B))]
5308   "TARGET_NEON"
5309 {
5310   int regno = REGNO (operands[0]);
5311   rtx ops[5];
5312   ops[0] = gen_rtx_REG (DImode, regno + 2);
5313   ops[1] = gen_rtx_REG (DImode, regno + 6);
5314   ops[2] = gen_rtx_REG (DImode, regno + 10);
5315   ops[3] = gen_rtx_REG (DImode, regno + 14);
5316   ops[4] = operands[1];
5317   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5318   return "";
5319 }
5320   [(set_attr "neon_type" "neon_vld3_vld4")]
5321 )
5322
5323 (define_insn "neon_vld4_lane<mode>"
5324   [(set (match_operand:OI 0 "s_register_operand" "=w")
5325         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5326                     (match_operand:OI 2 "s_register_operand" "0")
5327                     (match_operand:SI 3 "immediate_operand" "i")
5328                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5329                    UNSPEC_VLD4_LANE))]
5330   "TARGET_NEON"
5331 {
5332   HOST_WIDE_INT lane = INTVAL (operands[3]);
5333   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5334   int regno = REGNO (operands[0]);
5335   rtx ops[6];
5336   if (lane < 0 || lane >= max)
5337     error ("lane out of range");
5338   ops[0] = gen_rtx_REG (DImode, regno);
5339   ops[1] = gen_rtx_REG (DImode, regno + 2);
5340   ops[2] = gen_rtx_REG (DImode, regno + 4);
5341   ops[3] = gen_rtx_REG (DImode, regno + 6);
5342   ops[4] = operands[1];
5343   ops[5] = operands[3];
5344   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5345                    ops);
5346   return "";
5347 }
5348   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5349 )
5350
5351 (define_insn "neon_vld4_lane<mode>"
5352   [(set (match_operand:XI 0 "s_register_operand" "=w")
5353         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5354                     (match_operand:XI 2 "s_register_operand" "0")
5355                     (match_operand:SI 3 "immediate_operand" "i")
5356                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5357                    UNSPEC_VLD4_LANE))]
5358   "TARGET_NEON"
5359 {
5360   HOST_WIDE_INT lane = INTVAL (operands[3]);
5361   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5362   int regno = REGNO (operands[0]);
5363   rtx ops[6];
5364   if (lane < 0 || lane >= max)
5365     error ("lane out of range");
5366   else if (lane >= max / 2)
5367     {
5368       lane -= max / 2;
5369       regno += 2;
5370     }
5371   ops[0] = gen_rtx_REG (DImode, regno);
5372   ops[1] = gen_rtx_REG (DImode, regno + 4);
5373   ops[2] = gen_rtx_REG (DImode, regno + 8);
5374   ops[3] = gen_rtx_REG (DImode, regno + 12);
5375   ops[4] = operands[1];
5376   ops[5] = GEN_INT (lane);
5377   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5378                    ops);
5379   return "";
5380 }
5381   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5382 )
5383
5384 (define_insn "neon_vld4_dup<mode>"
5385   [(set (match_operand:OI 0 "s_register_operand" "=w")
5386         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5387                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5388                    UNSPEC_VLD4_DUP))]
5389   "TARGET_NEON"
5390 {
5391   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5392     {
5393       int regno = REGNO (operands[0]);
5394       rtx ops[5];
5395       ops[0] = gen_rtx_REG (DImode, regno);
5396       ops[1] = gen_rtx_REG (DImode, regno + 2);
5397       ops[2] = gen_rtx_REG (DImode, regno + 4);
5398       ops[3] = gen_rtx_REG (DImode, regno + 6);
5399       ops[4] = operands[1];
5400       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5401                        ops);
5402       return "";
5403     }
5404   else
5405     return "vld1.<V_sz_elem>\t%h0, %A1";
5406 }
5407   [(set (attr "neon_type")
5408       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5409                     (const_string "neon_vld3_vld4_all_lanes")
5410                     (const_string "neon_vld1_1_2_regs")))]
5411 )
5412
5413 (define_expand "vec_store_lanesoi<mode>"
5414   [(set (match_operand:OI 0 "neon_struct_operand")
5415         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5416                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5417                    UNSPEC_VST4))]
5418   "TARGET_NEON")
5419
5420 (define_insn "neon_vst4<mode>"
5421   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5422         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5423                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5424                    UNSPEC_VST4))]
5425   "TARGET_NEON"
5426 {
5427   if (<V_sz_elem> == 64)
5428     return "vst1.64\t%h1, %A0";
5429   else
5430     return "vst4.<V_sz_elem>\t%h1, %A0";
5431 }
5432   [(set (attr "neon_type")
5433       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5434                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5435                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5436 )
5437
5438 (define_expand "vec_store_lanesxi<mode>"
5439   [(match_operand:XI 0 "neon_struct_operand")
5440    (match_operand:XI 1 "s_register_operand")
5441    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5442   "TARGET_NEON"
5443 {
5444   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5445   DONE;
5446 })
5447
5448 (define_expand "neon_vst4<mode>"
5449   [(match_operand:XI 0 "neon_struct_operand")
5450    (match_operand:XI 1 "s_register_operand")
5451    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5452   "TARGET_NEON"
5453 {
5454   rtx mem;
5455
5456   mem = adjust_address (operands[0], OImode, 0);
5457   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5458   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5459   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5460   DONE;
5461 })
5462
5463 (define_insn "neon_vst4qa<mode>"
5464   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5465         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5466                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5467                    UNSPEC_VST4A))]
5468   "TARGET_NEON"
5469 {
5470   int regno = REGNO (operands[1]);
5471   rtx ops[5];
5472   ops[0] = operands[0];
5473   ops[1] = gen_rtx_REG (DImode, regno);
5474   ops[2] = gen_rtx_REG (DImode, regno + 4);
5475   ops[3] = gen_rtx_REG (DImode, regno + 8);
5476   ops[4] = gen_rtx_REG (DImode, regno + 12);
5477   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5478   return "";
5479 }
5480   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5481 )
5482
5483 (define_insn "neon_vst4qb<mode>"
5484   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5485         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5486                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5487                    UNSPEC_VST4B))]
5488   "TARGET_NEON"
5489 {
5490   int regno = REGNO (operands[1]);
5491   rtx ops[5];
5492   ops[0] = operands[0];
5493   ops[1] = gen_rtx_REG (DImode, regno + 2);
5494   ops[2] = gen_rtx_REG (DImode, regno + 6);
5495   ops[3] = gen_rtx_REG (DImode, regno + 10);
5496   ops[4] = gen_rtx_REG (DImode, regno + 14);
5497   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5498   return "";
5499 }
5500   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5501 )
5502
5503 (define_insn "neon_vst4_lane<mode>"
5504   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5505         (unspec:<V_four_elem>
5506            [(match_operand:OI 1 "s_register_operand" "w")
5507             (match_operand:SI 2 "immediate_operand" "i")
5508             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5509            UNSPEC_VST4_LANE))]
5510   "TARGET_NEON"
5511 {
5512   HOST_WIDE_INT lane = INTVAL (operands[2]);
5513   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5514   int regno = REGNO (operands[1]);
5515   rtx ops[6];
5516   if (lane < 0 || lane >= max)
5517     error ("lane out of range");
5518   ops[0] = operands[0];
5519   ops[1] = gen_rtx_REG (DImode, regno);
5520   ops[2] = gen_rtx_REG (DImode, regno + 2);
5521   ops[3] = gen_rtx_REG (DImode, regno + 4);
5522   ops[4] = gen_rtx_REG (DImode, regno + 6);
5523   ops[5] = operands[2];
5524   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5525                    ops);
5526   return "";
5527 }
5528   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5529 )
5530
5531 (define_insn "neon_vst4_lane<mode>"
5532   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5533         (unspec:<V_four_elem>
5534            [(match_operand:XI 1 "s_register_operand" "w")
5535             (match_operand:SI 2 "immediate_operand" "i")
5536             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5537            UNSPEC_VST4_LANE))]
5538   "TARGET_NEON"
5539 {
5540   HOST_WIDE_INT lane = INTVAL (operands[2]);
5541   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5542   int regno = REGNO (operands[1]);
5543   rtx ops[6];
5544   if (lane < 0 || lane >= max)
5545     error ("lane out of range");
5546   else if (lane >= max / 2)
5547     {
5548       lane -= max / 2;
5549       regno += 2;
5550     }
5551   ops[0] = operands[0];
5552   ops[1] = gen_rtx_REG (DImode, regno);
5553   ops[2] = gen_rtx_REG (DImode, regno + 4);
5554   ops[3] = gen_rtx_REG (DImode, regno + 8);
5555   ops[4] = gen_rtx_REG (DImode, regno + 12);
5556   ops[5] = GEN_INT (lane);
5557   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5558                    ops);
5559   return "";
5560 }
5561   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5562 )
5563
5564 (define_expand "neon_vand<mode>"
5565   [(match_operand:VDQX 0 "s_register_operand" "")
5566    (match_operand:VDQX 1 "s_register_operand" "")
5567    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5568    (match_operand:SI 3 "immediate_operand" "")]
5569   "TARGET_NEON"
5570 {
5571   emit_insn (gen_and<mode>3 (operands[0], operands[1], operands[2]));
5572   DONE;
5573 })
5574
5575 (define_expand "neon_vorr<mode>"
5576   [(match_operand:VDQX 0 "s_register_operand" "")
5577    (match_operand:VDQX 1 "s_register_operand" "")
5578    (match_operand:VDQX 2 "neon_logic_op2" "")
5579    (match_operand:SI 3 "immediate_operand" "")]
5580   "TARGET_NEON"
5581 {
5582   emit_insn (gen_ior<mode>3 (operands[0], operands[1], operands[2]));
5583   DONE;
5584 })
5585
5586 (define_expand "neon_veor<mode>"
5587   [(match_operand:VDQX 0 "s_register_operand" "")
5588    (match_operand:VDQX 1 "s_register_operand" "")
5589    (match_operand:VDQX 2 "s_register_operand" "")
5590    (match_operand:SI 3 "immediate_operand" "")]
5591   "TARGET_NEON"
5592 {
5593   emit_insn (gen_xor<mode>3 (operands[0], operands[1], operands[2]));
5594   DONE;
5595 })
5596
5597 (define_expand "neon_vbic<mode>"
5598   [(match_operand:VDQX 0 "s_register_operand" "")
5599    (match_operand:VDQX 1 "s_register_operand" "")
5600    (match_operand:VDQX 2 "neon_logic_op2" "")
5601    (match_operand:SI 3 "immediate_operand" "")]
5602   "TARGET_NEON"
5603 {
5604   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5605   DONE;
5606 })
5607
5608 (define_expand "neon_vorn<mode>"
5609   [(match_operand:VDQX 0 "s_register_operand" "")
5610    (match_operand:VDQX 1 "s_register_operand" "")
5611    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5612    (match_operand:SI 3 "immediate_operand" "")]
5613   "TARGET_NEON"
5614 {
5615   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5616   DONE;
5617 })
5618
5619 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5620   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5621         (SE:<V_unpack> (vec_select:<V_HALF>
5622                           (match_operand:VU 1 "register_operand" "w")
5623                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5624   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5625   "vmovl.<US><V_sz_elem> %q0, %e1"
5626   [(set_attr "neon_type" "neon_shift_1")]
5627 )
5628
5629 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5630   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5631         (SE:<V_unpack> (vec_select:<V_HALF>
5632                           (match_operand:VU 1 "register_operand" "w")
5633                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5634   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5635   "vmovl.<US><V_sz_elem> %q0, %f1"
5636   [(set_attr "neon_type" "neon_shift_1")]
5637 )
5638
5639 (define_expand "vec_unpack<US>_hi_<mode>"
5640   [(match_operand:<V_unpack> 0 "register_operand" "")
5641    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5642  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5643   {
5644    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5645    rtx t1;
5646    int i;
5647    for (i = 0; i < (<V_mode_nunits>/2); i++)
5648      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5649
5650    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5651    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5652                                                  operands[1],
5653                                                  t1));
5654    DONE;
5655   }
5656 )
5657
5658 (define_expand "vec_unpack<US>_lo_<mode>"
5659   [(match_operand:<V_unpack> 0 "register_operand" "")
5660    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5661  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5662   {
5663    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5664    rtx t1;
5665    int i;
5666    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5667      RTVEC_ELT (v, i) = GEN_INT (i);
5668    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5669    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5670                                                  operands[1],
5671                                                  t1));
5672    DONE;
5673   }
5674 )
5675
5676 (define_insn "neon_vec_<US>mult_lo_<mode>"
5677  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5678        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5679                            (match_operand:VU 1 "register_operand" "w")
5680                            (match_operand:VU 2 "vect_par_constant_low" "")))
5681                         (SE:<V_unpack> (vec_select:<V_HALF>
5682                            (match_operand:VU 3 "register_operand" "w")
5683                            (match_dup 2)))))]
5684   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5685   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5686   [(set_attr "neon_type" "neon_shift_1")]
5687 )
5688
5689 (define_expand "vec_widen_<US>mult_lo_<mode>"
5690   [(match_operand:<V_unpack> 0 "register_operand" "")
5691    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5692    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5693  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5694  {
5695    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5696    rtx t1;
5697    int i;
5698    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5699      RTVEC_ELT (v, i) = GEN_INT (i);
5700    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5701
5702    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5703                                                operands[1],
5704                                                t1,
5705                                                operands[2]));
5706    DONE;
5707  }
5708 )
5709
5710 (define_insn "neon_vec_<US>mult_hi_<mode>"
5711  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5712       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5713                             (match_operand:VU 1 "register_operand" "w")
5714                             (match_operand:VU 2 "vect_par_constant_high" "")))
5715                        (SE:<V_unpack> (vec_select:<V_HALF>
5716                             (match_operand:VU 3 "register_operand" "w")
5717                             (match_dup 2)))))]
5718   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5719   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5720   [(set_attr "neon_type" "neon_shift_1")]
5721 )
5722
5723 (define_expand "vec_widen_<US>mult_hi_<mode>"
5724   [(match_operand:<V_unpack> 0 "register_operand" "")
5725    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5726    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5727  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5728  {
5729    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5730    rtx t1;
5731    int i;
5732    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5733      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5734    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5735
5736    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5737                                                operands[1],
5738                                                t1,
5739                                                operands[2]));
5740    DONE;
5741
5742  }
5743 )
5744
5745 (define_insn "neon_vec_<US>shiftl_<mode>"
5746  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5747        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5748        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5749   "TARGET_NEON"
5750 {
5751   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5752 }
5753   [(set_attr "neon_type" "neon_shift_1")]
5754 )
5755
5756 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5757   [(match_operand:<V_unpack> 0 "register_operand" "")
5758    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5759    (match_operand:SI 2 "immediate_operand" "i")]
5760  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5761  {
5762   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5763                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5764                 operands[2]));
5765    DONE;
5766  }
5767 )
5768
5769 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5770   [(match_operand:<V_unpack> 0 "register_operand" "")
5771    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5772    (match_operand:SI 2 "immediate_operand" "i")]
5773  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5774  {
5775   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5776                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5777                                      GET_MODE_SIZE (<V_HALF>mode)),
5778                 operands[2]));
5779    DONE;
5780  }
5781 )
5782
5783 ;; Vectorize for non-neon-quad case
5784 (define_insn "neon_unpack<US>_<mode>"
5785  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5786        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5787  "TARGET_NEON"
5788  "vmovl.<US><V_sz_elem> %q0, %P1"
5789   [(set_attr "neon_type" "neon_shift_1")]
5790 )
5791
5792 (define_expand "vec_unpack<US>_lo_<mode>"
5793  [(match_operand:<V_double_width> 0 "register_operand" "")
5794   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5795  "TARGET_NEON"
5796 {
5797   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5798   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5799   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5800
5801   DONE;
5802 }
5803 )
5804
5805 (define_expand "vec_unpack<US>_hi_<mode>"
5806  [(match_operand:<V_double_width> 0 "register_operand" "")
5807   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5808  "TARGET_NEON"
5809 {
5810   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5811   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5812   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5813
5814   DONE;
5815 }
5816 )
5817
5818 (define_insn "neon_vec_<US>mult_<mode>"
5819  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5820        (mult:<V_widen> (SE:<V_widen>
5821                            (match_operand:VDI 1 "register_operand" "w"))
5822                        (SE:<V_widen>
5823                            (match_operand:VDI 2 "register_operand" "w"))))]
5824   "TARGET_NEON"
5825   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5826   [(set_attr "neon_type" "neon_shift_1")]
5827 )
5828
5829 (define_expand "vec_widen_<US>mult_hi_<mode>"
5830   [(match_operand:<V_double_width> 0 "register_operand" "")
5831    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5832    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5833  "TARGET_NEON"
5834  {
5835    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5836    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5837    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5838
5839    DONE;
5840
5841  }
5842 )
5843
5844 (define_expand "vec_widen_<US>mult_lo_<mode>"
5845   [(match_operand:<V_double_width> 0 "register_operand" "")
5846    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5847    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5848  "TARGET_NEON"
5849  {
5850    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5851    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5852    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5853
5854    DONE;
5855
5856  }
5857 )
5858
5859 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5860  [(match_operand:<V_double_width> 0 "register_operand" "")
5861    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5862    (match_operand:SI 2 "immediate_operand" "i")]
5863  "TARGET_NEON"
5864  {
5865    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5866    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5867    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5868
5869    DONE;
5870  }
5871 )
5872
5873 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5874   [(match_operand:<V_double_width> 0 "register_operand" "")
5875    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5876    (match_operand:SI 2 "immediate_operand" "i")]
5877  "TARGET_NEON"
5878  {
5879    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5880    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5881    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5882
5883    DONE;
5884  }
5885 )
5886
5887 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5888 ; because the ordering of vector elements in Q registers is different from what
5889 ; the semantics of the instructions require.
5890
5891 (define_insn "vec_pack_trunc_<mode>"
5892  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5893        (vec_concat:<V_narrow_pack>
5894                 (truncate:<V_narrow>
5895                         (match_operand:VN 1 "register_operand" "w"))
5896                 (truncate:<V_narrow>
5897                         (match_operand:VN 2 "register_operand" "w"))))]
5898  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5899  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5900  [(set_attr "neon_type" "neon_shift_1")
5901   (set_attr "length" "8")]
5902 )
5903
5904 ;; For the non-quad case.
5905 (define_insn "neon_vec_pack_trunc_<mode>"
5906  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5907        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5908  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5909  "vmovn.i<V_sz_elem>\t%P0, %q1"
5910  [(set_attr "neon_type" "neon_shift_1")]
5911 )
5912
5913 (define_expand "vec_pack_trunc_<mode>"
5914  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5915   (match_operand:VSHFT 1 "register_operand" "")
5916   (match_operand:VSHFT 2 "register_operand")]
5917  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5918 {
5919   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5920
5921   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5922   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5923   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5924   DONE;
5925 })
5926
5927 (define_insn "neon_vabd<mode>_2"
5928  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5929        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5930                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5931  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5932  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5933  [(set (attr "neon_type")
5934        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5935                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5936                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5937                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5938                      (const_string "neon_int_5")))]
5939 )
5940
5941 (define_insn "neon_vabd<mode>_3"
5942  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5943        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5944                              (match_operand:VDQ 2 "s_register_operand" "w")]
5945                  UNSPEC_VSUB)))]
5946  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5947  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5948  [(set (attr "neon_type")
5949        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5950                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5951                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5952                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5953                      (const_string "neon_int_5")))]
5954 )
5955
5956 ;; Copy from core-to-neon regs, then extend, not vice-versa
5957
5958 (define_split
5959   [(set (match_operand:DI 0 "s_register_operand" "")
5960         (sign_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5961   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5962   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5963    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 32)))]
5964   {
5965     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5966   })
5967
5968 (define_split
5969   [(set (match_operand:DI 0 "s_register_operand" "")
5970         (sign_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5971   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5972   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5973    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 48)))]
5974   {
5975     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5976   })
5977
5978 (define_split
5979   [(set (match_operand:DI 0 "s_register_operand" "")
5980         (sign_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5981   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5982   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5983    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))]
5984   {
5985     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5986   })
5987
5988 (define_split
5989   [(set (match_operand:DI 0 "s_register_operand" "")
5990         (zero_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5991   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5992   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5993    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 32)))]
5994   {
5995     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5996   })
5997
5998 (define_split
5999   [(set (match_operand:DI 0 "s_register_operand" "")
6000         (zero_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
6001   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
6002   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
6003    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 48)))]
6004   {
6005     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
6006   })
6007
6008 (define_split
6009   [(set (match_operand:DI 0 "s_register_operand" "")
6010         (zero_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
6011   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
6012   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
6013    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
6014   {
6015     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
6016   })