gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012
   3 ;;   Free Software Foundation, Inc.
   4 ;; Written by CodeSourcery.
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published by
  10 ;; the Free Software Foundation; either version 3, or (at your option)
  11 ;; any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but
  14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 ;; General Public License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22
  23 ;; Attribute used to permit string comparisons against <VQH_mnem> in
  24 ;; neon_type attribute definitions.
  25 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
  26
  27 (define_insn "*neon_mov<mode>"
  28   [(set (match_operand:VDX 0 "nonimmediate_operand"
  29           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
  30         (match_operand:VDX 1 "general_operand"
  31           " w,w, Dn,Uni, w, r, r, Usi,r"))]
  32   "TARGET_NEON
  33    && (register_operand (operands[0], <MODE>mode)
  34        || register_operand (operands[1], <MODE>mode))"
  35 {
  36   if (which_alternative == 2)
  37     {
  38       int width, is_valid;
  39       static char templ[40];
  40
  41       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  42         &operands[1], &width);
  43
  44       gcc_assert (is_valid != 0);
  45
  46       if (width == 0)
  47         return "vmov.f32\t%P0, %1  @ <mode>";
  48       else
  49         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
  50
  51       return templ;
  52     }
  53
  54   switch (which_alternative)
  55     {
  56     case 0: return "vmov\t%P0, %P1  @ <mode>";
  57     case 1: case 3: return output_move_neon (operands);
  58     case 2: gcc_unreachable ();
  59     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
  60     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
  61     default: return output_move_double (operands, true, NULL);
  62     }
  63 }
  64  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
  65   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu_reg,load2,store2")
  66   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
  67   (set_attr "length" "4,4,4,4,4,4,8,8,8")
  68   (set_attr "arm_pool_range"     "*,*,*,1020,*,*,*,1020,*")
  69   (set_attr "thumb2_pool_range"     "*,*,*,1018,*,*,*,1018,*")
  70   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
  71
  72 (define_insn "*neon_mov<mode>"
  73   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
  74           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
  75         (match_operand:VQXMOV 1 "general_operand"
  76           " w,w, Dn,Uni, w, r, r, Usi, r"))]
  77   "TARGET_NEON
  78    && (register_operand (operands[0], <MODE>mode)
  79        || register_operand (operands[1], <MODE>mode))"
  80 {
  81   if (which_alternative == 2)
  82     {
  83       int width, is_valid;
  84       static char templ[40];
  85
  86       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  87         &operands[1], &width);
  88
  89       gcc_assert (is_valid != 0);
  90
  91       if (width == 0)
  92         return "vmov.f32\t%q0, %1  @ <mode>";
  93       else
  94         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
  95
  96       return templ;
  97     }
  98
  99   switch (which_alternative)
 100     {
 101     case 0: return "vmov\t%q0, %q1  @ <mode>";
 102     case 1: case 3: return output_move_neon (operands);
 103     case 2: gcc_unreachable ();
 104     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 105     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 106     default: return output_move_quad (operands);
 107     }
 108 }
 109   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 110                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 111    (set_attr "type" "*,*,*,*,*,*,alu_reg,load4,store4")
 112    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 113    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 114    (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*")
 115    (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*")
 116    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 117
 118 (define_expand "movti"
 119   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 120         (match_operand:TI 1 "general_operand" ""))]
 121   "TARGET_NEON"
 122 {
 123   if (can_create_pseudo_p ())
 124     {
 125       if (!REG_P (operands[0]))
 126         operands[1] = force_reg (TImode, operands[1]);
 127     }
 128 })
 129
 130 (define_expand "mov<mode>"
 131   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 132         (match_operand:VSTRUCT 1 "general_operand" ""))]
 133   "TARGET_NEON"
 134 {
 135   if (can_create_pseudo_p ())
 136     {
 137       if (!REG_P (operands[0]))
 138         operands[1] = force_reg (<MODE>mode, operands[1]);
 139     }
 140 })
 141
 142 (define_insn "*neon_mov<mode>"
 143   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 144         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 145   "TARGET_NEON
 146    && (register_operand (operands[0], <MODE>mode)
 147        || register_operand (operands[1], <MODE>mode))"
 148 {
 149   switch (which_alternative)
 150     {
 151     case 0: return "#";
 152     case 1: case 2: return output_move_neon (operands);
 153     default: gcc_unreachable ();
 154     }
 155 }
 156   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 157    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 158
 159 (define_split
 160   [(set (match_operand:EI 0 "s_register_operand" "")
 161         (match_operand:EI 1 "s_register_operand" ""))]
 162   "TARGET_NEON && reload_completed"
 163   [(set (match_dup 0) (match_dup 1))
 164    (set (match_dup 2) (match_dup 3))]
 165 {
 166   int rdest = REGNO (operands[0]);
 167   int rsrc = REGNO (operands[1]);
 168   rtx dest[2], src[2];
 169
 170   dest[0] = gen_rtx_REG (TImode, rdest);
 171   src[0] = gen_rtx_REG (TImode, rsrc);
 172   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 173   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 174
 175   neon_disambiguate_copy (operands, dest, src, 2);
 176 })
 177
 178 (define_split
 179   [(set (match_operand:OI 0 "s_register_operand" "")
 180         (match_operand:OI 1 "s_register_operand" ""))]
 181   "TARGET_NEON && reload_completed"
 182   [(set (match_dup 0) (match_dup 1))
 183    (set (match_dup 2) (match_dup 3))]
 184 {
 185   int rdest = REGNO (operands[0]);
 186   int rsrc = REGNO (operands[1]);
 187   rtx dest[2], src[2];
 188
 189   dest[0] = gen_rtx_REG (TImode, rdest);
 190   src[0] = gen_rtx_REG (TImode, rsrc);
 191   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 192   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 193
 194   neon_disambiguate_copy (operands, dest, src, 2);
 195 })
 196
 197 (define_split
 198   [(set (match_operand:CI 0 "s_register_operand" "")
 199         (match_operand:CI 1 "s_register_operand" ""))]
 200   "TARGET_NEON && reload_completed"
 201   [(set (match_dup 0) (match_dup 1))
 202    (set (match_dup 2) (match_dup 3))
 203    (set (match_dup 4) (match_dup 5))]
 204 {
 205   int rdest = REGNO (operands[0]);
 206   int rsrc = REGNO (operands[1]);
 207   rtx dest[3], src[3];
 208
 209   dest[0] = gen_rtx_REG (TImode, rdest);
 210   src[0] = gen_rtx_REG (TImode, rsrc);
 211   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 212   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 213   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 214   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 215
 216   neon_disambiguate_copy (operands, dest, src, 3);
 217 })
 218
 219 (define_split
 220   [(set (match_operand:XI 0 "s_register_operand" "")
 221         (match_operand:XI 1 "s_register_operand" ""))]
 222   "TARGET_NEON && reload_completed"
 223   [(set (match_dup 0) (match_dup 1))
 224    (set (match_dup 2) (match_dup 3))
 225    (set (match_dup 4) (match_dup 5))
 226    (set (match_dup 6) (match_dup 7))]
 227 {
 228   int rdest = REGNO (operands[0]);
 229   int rsrc = REGNO (operands[1]);
 230   rtx dest[4], src[4];
 231
 232   dest[0] = gen_rtx_REG (TImode, rdest);
 233   src[0] = gen_rtx_REG (TImode, rsrc);
 234   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 235   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 236   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 237   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 238   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 239   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 240
 241   neon_disambiguate_copy (operands, dest, src, 4);
 242 })
 243
 244 (define_expand "movmisalign<mode>"
 245   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 246         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 247                      UNSPEC_MISALIGNED_ACCESS))]
 248   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 249 {
 250   /* This pattern is not permitted to fail during expansion: if both arguments
 251      are non-registers (e.g. memory := constant, which can be created by the
 252      auto-vectorizer), force operand 1 into a register.  */
 253   if (!s_register_operand (operands[0], <MODE>mode)
 254       && !s_register_operand (operands[1], <MODE>mode))
 255     operands[1] = force_reg (<MODE>mode, operands[1]);
 256 })
 257
 258 (define_insn "*movmisalign<mode>_neon_store"
 259   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 260         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 261                     UNSPEC_MISALIGNED_ACCESS))]
 262   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 263   "vst1.<V_sz_elem>\t{%P1}, %A0"
 264   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 265
 266 (define_insn "*movmisalign<mode>_neon_load"
 267   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 268         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 269                     UNSPEC_MISALIGNED_ACCESS))]
 270   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 271   "vld1.<V_sz_elem>\t{%P0}, %A1"
 272   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 273
 274 (define_insn "*movmisalign<mode>_neon_store"
 275   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 276         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 277                     UNSPEC_MISALIGNED_ACCESS))]
 278   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 279   "vst1.<V_sz_elem>\t{%q1}, %A0"
 280   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 281
 282 (define_insn "*movmisalign<mode>_neon_load"
 283   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 284         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 285                     UNSPEC_MISALIGNED_ACCESS))]
 286   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 287   "vld1.<V_sz_elem>\t{%q0}, %A1"
 288   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 289
 290 (define_insn "vec_set<mode>_internal"
 291   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 292         (vec_merge:VD
 293           (vec_duplicate:VD
 294             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 295           (match_operand:VD 3 "s_register_operand" "0,0")
 296           (match_operand:SI 2 "immediate_operand" "i,i")))]
 297   "TARGET_NEON"
 298 {
 299   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 300   if (BYTES_BIG_ENDIAN)
 301     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 302   operands[2] = GEN_INT (elt);
 303
 304   if (which_alternative == 0)
 305     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 306   else
 307     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 308 }
 309   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")])
 310
 311 (define_insn "vec_set<mode>_internal"
 312   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 313         (vec_merge:VQ
 314           (vec_duplicate:VQ
 315             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 316           (match_operand:VQ 3 "s_register_operand" "0,0")
 317           (match_operand:SI 2 "immediate_operand" "i,i")))]
 318   "TARGET_NEON"
 319 {
 320   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 321   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 322   int elt = elem % half_elts;
 323   int hi = (elem / half_elts) * 2;
 324   int regno = REGNO (operands[0]);
 325
 326   if (BYTES_BIG_ENDIAN)
 327     elt = half_elts - 1 - elt;
 328
 329   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 330   operands[2] = GEN_INT (elt);
 331
 332   if (which_alternative == 0)
 333     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 334   else
 335     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 336 }
 337   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
 338 )
 339
 340 (define_insn "vec_setv2di_internal"
 341   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 342         (vec_merge:V2DI
 343           (vec_duplicate:V2DI
 344             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 345           (match_operand:V2DI 3 "s_register_operand" "0,0")
 346           (match_operand:SI 2 "immediate_operand" "i,i")))]
 347   "TARGET_NEON"
 348 {
 349   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 350   int regno = REGNO (operands[0]) + 2 * elem;
 351
 352   operands[0] = gen_rtx_REG (DImode, regno);
 353
 354   if (which_alternative == 0)
 355     return "vld1.64\t%P0, %A1";
 356   else
 357     return "vmov\t%P0, %Q1, %R1";
 358 }
 359   [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")]
 360 )
 361
 362 (define_expand "vec_set<mode>"
 363   [(match_operand:VDQ 0 "s_register_operand" "")
 364    (match_operand:<V_elem> 1 "s_register_operand" "")
 365    (match_operand:SI 2 "immediate_operand" "")]
 366   "TARGET_NEON"
 367 {
 368   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 369   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 370                                          GEN_INT (elem), operands[0]));
 371   DONE;
 372 })
 373
 374 (define_insn "vec_extract<mode>"
 375   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 376         (vec_select:<V_elem>
 377           (match_operand:VD 1 "s_register_operand" "w,w")
 378           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 379   "TARGET_NEON"
 380 {
 381   if (BYTES_BIG_ENDIAN)
 382     {
 383       int elt = INTVAL (operands[2]);
 384       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 385       operands[2] = GEN_INT (elt);
 386     }
 387
 388   if (which_alternative == 0)
 389     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 390   else
 391     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 392 }
 393   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 394 )
 395
 396 (define_insn "vec_extract<mode>"
 397   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 398         (vec_select:<V_elem>
 399           (match_operand:VQ 1 "s_register_operand" "w,w")
 400           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 401   "TARGET_NEON"
 402 {
 403   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 404   int elt = INTVAL (operands[2]) % half_elts;
 405   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 406   int regno = REGNO (operands[1]);
 407
 408   if (BYTES_BIG_ENDIAN)
 409     elt = half_elts - 1 - elt;
 410
 411   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 412   operands[2] = GEN_INT (elt);
 413
 414   if (which_alternative == 0)
 415     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 416   else
 417     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 418 }
 419   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 420 )
 421
 422 (define_insn "vec_extractv2di"
 423   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 424         (vec_select:DI
 425           (match_operand:V2DI 1 "s_register_operand" "w,w")
 426           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 427   "TARGET_NEON"
 428 {
 429   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 430
 431   operands[1] = gen_rtx_REG (DImode, regno);
 432
 433   if (which_alternative == 0)
 434     return "vst1.64\t{%P1}, %A0  @ v2di";
 435   else
 436     return "vmov\t%Q0, %R0, %P1  @ v2di";
 437 }
 438   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")]
 439 )
 440
 441 (define_expand "vec_init<mode>"
 442   [(match_operand:VDQ 0 "s_register_operand" "")
 443    (match_operand 1 "" "")]
 444   "TARGET_NEON"
 445 {
 446   neon_expand_vector_init (operands[0], operands[1]);
 447   DONE;
 448 })
 449
 450 ;; Doubleword and quadword arithmetic.
 451
 452 ;; NOTE: some other instructions also support 64-bit integer
 453 ;; element size, which we could potentially use for "long long" operations.
 454
 455 (define_insn "*add<mode>3_neon"
 456   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 457         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 458                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 459   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 460   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 461   [(set (attr "neon_type")
 462       (if_then_else (match_test "<Is_float_mode>")
 463                     (if_then_else (match_test "<Is_d_reg>")
 464                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 465                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 466                     (const_string "neon_int_1")))]
 467 )
 468
 469 (define_insn "adddi3_neon"
 470   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 471         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 472                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 473    (clobber (reg:CC CC_REGNUM))]
 474   "TARGET_NEON"
 475 {
 476   switch (which_alternative)
 477     {
 478     case 0: /* fall through */
 479     case 3: return "vadd.i64\t%P0, %P1, %P2";
 480     case 1: return "#";
 481     case 2: return "#";
 482     case 4: return "#";
 483     case 5: return "#";
 484     case 6: return "#";
 485     default: gcc_unreachable ();
 486     }
 487 }
 488   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 489    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 490    (set_attr "length" "*,8,8,*,8,8,8")
 491    (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
 492 )
 493
 494 (define_insn "*sub<mode>3_neon"
 495   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 496         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 497                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 498   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 499   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 500   [(set (attr "neon_type")
 501       (if_then_else (match_test "<Is_float_mode>")
 502                     (if_then_else (match_test "<Is_d_reg>")
 503                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 504                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 505                     (const_string "neon_int_2")))]
 506 )
 507
 508 (define_insn "subdi3_neon"
 509   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 510         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 511                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 512    (clobber (reg:CC CC_REGNUM))]
 513   "TARGET_NEON"
 514 {
 515   switch (which_alternative)
 516     {
 517     case 0: /* fall through */
 518     case 4: return "vsub.i64\t%P0, %P1, %P2";
 519     case 1: /* fall through */
 520     case 2: /* fall through */
 521     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 522     default: gcc_unreachable ();
 523     }
 524 }
 525   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 526    (set_attr "conds" "*,clob,clob,clob,*")
 527    (set_attr "length" "*,8,8,8,*")
 528    (set_attr "arch" "nota8,*,*,*,onlya8")]
 529 )
 530
 531 (define_insn "*mul<mode>3_neon"
 532   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 533         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 534                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 535   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 536   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 537   [(set (attr "neon_type")
 538       (if_then_else (match_test "<Is_float_mode>")
 539                     (if_then_else (match_test "<Is_d_reg>")
 540                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 541                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 542                     (if_then_else (match_test "<Is_d_reg>")
 543                                   (if_then_else
 544                                     (match_test "<Scalar_mul_8_16>")
 545                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 546                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 547                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 548                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 549                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 550 )
 551
 552 (define_insn "mul<mode>3add<mode>_neon"
 553   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 554         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 555                             (match_operand:VDQ 3 "s_register_operand" "w"))
 556                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 557   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 558   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 559   [(set (attr "neon_type")
 560       (if_then_else (match_test "<Is_float_mode>")
 561                     (if_then_else (match_test "<Is_d_reg>")
 562                                   (const_string "neon_fp_vmla_ddd")
 563                                   (const_string "neon_fp_vmla_qqq"))
 564                     (if_then_else (match_test "<Is_d_reg>")
 565                                   (if_then_else
 566                                     (match_test "<Scalar_mul_8_16>")
 567                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 568                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 569                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 570                                     (const_string "neon_mla_qqq_8_16")
 571                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 572 )
 573
 574 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 575   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 576         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 577                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 578                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 579   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 580   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 581   [(set (attr "neon_type")
 582       (if_then_else (match_test "<Is_float_mode>")
 583                     (if_then_else (match_test "<Is_d_reg>")
 584                                   (const_string "neon_fp_vmla_ddd")
 585                                   (const_string "neon_fp_vmla_qqq"))
 586                     (if_then_else (match_test "<Is_d_reg>")
 587                                   (if_then_else
 588                                     (match_test "<Scalar_mul_8_16>")
 589                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 590                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 591                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 592                                     (const_string "neon_mla_qqq_8_16")
 593                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 594 )
 595
 596 ;; Fused multiply-accumulate
 597 ;; We define each insn twice here:
 598 ;;    1: with flag_unsafe_math_optimizations for the widening multiply phase
 599 ;;       to be able to use when converting to FMA.
 600 ;;    2: without flag_unsafe_math_optimizations for the intrinsics to use.
 601 (define_insn "fma<VCVTF:mode>4"
 602   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 603         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 604                  (match_operand:VCVTF 2 "register_operand" "w")
 605                  (match_operand:VCVTF 3 "register_operand" "0")))]
 606   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 607   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 608   [(set (attr "neon_type")
 609         (if_then_else (match_test "<Is_d_reg>")
 610                       (const_string "neon_fp_vmla_ddd")
 611                       (const_string "neon_fp_vmla_qqq")))]
 612 )
 613
 614 (define_insn "fma<VCVTF:mode>4_intrinsic"
 615   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 616         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 617                  (match_operand:VCVTF 2 "register_operand" "w")
 618                  (match_operand:VCVTF 3 "register_operand" "0")))]
 619   "TARGET_NEON && TARGET_FMA"
 620   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 621   [(set (attr "neon_type")
 622         (if_then_else (match_test "<Is_d_reg>")
 623                       (const_string "neon_fp_vmla_ddd")
 624                       (const_string "neon_fp_vmla_qqq")))]
 625 )
 626
 627 (define_insn "*fmsub<VCVTF:mode>4"
 628   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 629         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 630                    (match_operand:VCVTF 2 "register_operand" "w")
 631                    (match_operand:VCVTF 3 "register_operand" "0")))]
 632   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 633   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 634   [(set (attr "neon_type")
 635         (if_then_else (match_test "<Is_d_reg>")
 636                       (const_string "neon_fp_vmla_ddd")
 637                       (const_string "neon_fp_vmla_qqq")))]
 638 )
 639
 640 (define_insn "fmsub<VCVTF:mode>4_intrinsic"
 641   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 642         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 643                    (match_operand:VCVTF 2 "register_operand" "w")
 644                    (match_operand:VCVTF 3 "register_operand" "0")))]
 645   "TARGET_NEON && TARGET_FMA"
 646   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 647   [(set (attr "neon_type")
 648         (if_then_else (match_test "<Is_d_reg>")
 649                       (const_string "neon_fp_vmla_ddd")
 650                       (const_string "neon_fp_vmla_qqq")))]
 651 )
 652
 653 (define_insn "neon_vrint<NEON_VRINT:nvrint_variant><VCVTF:mode>"
 654   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
 655         (unspec:VCVTF [(match_operand:VCVTF 1
 656                          "s_register_operand" "w")]
 657                 NEON_VRINT))]
 658   "TARGET_NEON && TARGET_FPU_ARMV8"
 659   "vrint<nvrint_variant>%?.f32\\t%<V_reg>0, %<V_reg>1"
 660   [(set (attr "neon_type")
 661         (if_then_else (match_test "<Is_d_reg>")
 662                 (const_string "neon_fp_vadd_ddd_vabs_dd")
 663                 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
 664 )
 665
 666 (define_insn "ior<mode>3"
 667   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 668         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 669                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 670   "TARGET_NEON"
 671 {
 672   switch (which_alternative)
 673     {
 674     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 675     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 676                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 677     default: gcc_unreachable ();
 678     }
 679 }
 680   [(set_attr "neon_type" "neon_int_1")]
 681 )
 682
 683 (define_insn "iordi3_neon"
 684   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 685         (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 686                 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
 687   "TARGET_NEON"
 688 {
 689   switch (which_alternative)
 690     {
 691     case 0: /* fall through */
 692     case 4: return "vorr\t%P0, %P1, %P2";
 693     case 1: /* fall through */
 694     case 5: return neon_output_logic_immediate ("vorr", &operands[2],
 695                      DImode, 0, VALID_NEON_QREG_MODE (DImode));
 696     case 2: return "#";
 697     case 3: return "#";
 698     default: gcc_unreachable ();
 699     }
 700 }
 701   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 702    (set_attr "length" "*,*,8,8,*,*")
 703    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 704 )
 705
 706 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 707 ;; vorr. We support the pseudo-instruction vand instead, because that
 708 ;; corresponds to the canonical form the middle-end expects to use for
 709 ;; immediate bitwise-ANDs.
 710
 711 (define_insn "and<mode>3"
 712   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 713         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 714                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 715   "TARGET_NEON"
 716 {
 717   switch (which_alternative)
 718     {
 719     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 720     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 721                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 722     default: gcc_unreachable ();
 723     }
 724 }
 725   [(set_attr "neon_type" "neon_int_1")]
 726 )
 727
 728 (define_insn "anddi3_neon"
 729   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 730         (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 731                 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
 732   "TARGET_NEON"
 733 {
 734   switch (which_alternative)
 735     {
 736     case 0: /* fall through */
 737     case 4: return "vand\t%P0, %P1, %P2";
 738     case 1: /* fall through */
 739     case 5: return neon_output_logic_immediate ("vand", &operands[2],
 740                      DImode, 1, VALID_NEON_QREG_MODE (DImode));
 741     case 2: return "#";
 742     case 3: return "#";
 743     default: gcc_unreachable ();
 744     }
 745 }
 746   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 747    (set_attr "length" "*,*,8,8,*,*")
 748    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 749 )
 750
 751 (define_insn "orn<mode>3_neon"
 752   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 753         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 754                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 755   "TARGET_NEON"
 756   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 757   [(set_attr "neon_type" "neon_int_1")]
 758 )
 759
 760 ;; TODO: investigate whether we should disable
 761 ;; this and bicdi3_neon for the A8 in line with the other
 762 ;; changes above.
 763 (define_insn_and_split "orndi3_neon"
 764   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 765         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 766                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 767   "TARGET_NEON"
 768   "@
 769    vorn\t%P0, %P1, %P2
 770    #
 771    #
 772    #"
 773   "reload_completed &&
 774    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 775   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 776    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 777   "
 778   {
 779     if (TARGET_THUMB2)
 780       {
 781         operands[3] = gen_highpart (SImode, operands[0]);
 782         operands[0] = gen_lowpart (SImode, operands[0]);
 783         operands[4] = gen_highpart (SImode, operands[2]);
 784         operands[2] = gen_lowpart (SImode, operands[2]);
 785         operands[5] = gen_highpart (SImode, operands[1]);
 786         operands[1] = gen_lowpart (SImode, operands[1]);
 787       }
 788     else
 789       {
 790         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 791         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 792         DONE;
 793       }
 794   }"
 795   [(set_attr "neon_type" "neon_int_1,*,*,*")
 796    (set_attr "length" "*,16,8,8")
 797    (set_attr "arch" "any,a,t2,t2")]
 798 )
 799
 800 (define_insn "bic<mode>3_neon"
 801   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 802         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 803                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 804   "TARGET_NEON"
 805   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 806   [(set_attr "neon_type" "neon_int_1")]
 807 )
 808
 809 ;; Compare to *anddi_notdi_di.
 810 (define_insn "bicdi3_neon"
 811   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 812         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 813                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 814   "TARGET_NEON"
 815   "@
 816    vbic\t%P0, %P1, %P2
 817    #
 818    #"
 819   [(set_attr "neon_type" "neon_int_1,*,*")
 820    (set_attr "length" "*,8,8")]
 821 )
 822
 823 (define_insn "xor<mode>3"
 824   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 825         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 826                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 827   "TARGET_NEON"
 828   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 829   [(set_attr "neon_type" "neon_int_1")]
 830 )
 831
 832 (define_insn "xordi3_neon"
 833   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
 834         (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
 835                 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
 836   "TARGET_NEON"
 837   "@
 838    veor\t%P0, %P1, %P2
 839    #
 840    #
 841    veor\t%P0, %P1, %P2"
 842   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
 843    (set_attr "length" "*,8,8,*")
 844    (set_attr "arch" "nota8,*,*,onlya8")]
 845 )
 846
 847 (define_insn "one_cmpl<mode>2"
 848   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 849         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 850   "TARGET_NEON"
 851   "vmvn\t%<V_reg>0, %<V_reg>1"
 852   [(set_attr "neon_type" "neon_int_1")]
 853 )
 854
 855 (define_insn "abs<mode>2"
 856   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 857         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 858   "TARGET_NEON"
 859   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 860   [(set (attr "neon_type")
 861       (if_then_else (match_test "<Is_float_mode>")
 862                     (if_then_else (match_test "<Is_d_reg>")
 863                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 864                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 865                     (const_string "neon_int_3")))]
 866 )
 867
 868 (define_insn "neg<mode>2"
 869   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 870         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 871   "TARGET_NEON"
 872   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 873   [(set (attr "neon_type")
 874       (if_then_else (match_test "<Is_float_mode>")
 875                     (if_then_else (match_test "<Is_d_reg>")
 876                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 877                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 878                     (const_string "neon_int_3")))]
 879 )
 880
 881 (define_insn "negdi2_neon"
 882   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 883         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 884    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 885    (clobber (reg:CC CC_REGNUM))]
 886   "TARGET_NEON"
 887   "#"
 888   [(set_attr "length" "8")]
 889 )
 890
 891 ; Split negdi2_neon for vfp registers
 892 (define_split
 893   [(set (match_operand:DI 0 "s_register_operand" "")
 894         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 895    (clobber (match_scratch:DI 2 ""))
 896    (clobber (reg:CC CC_REGNUM))]
 897   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 898   [(set (match_dup 2) (const_int 0))
 899    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 900               (clobber (reg:CC CC_REGNUM))])]
 901   {
 902     if (!REG_P (operands[2]))
 903       operands[2] = operands[0];
 904   }
 905 )
 906
 907 ; Split negdi2_neon for core registers
 908 (define_split
 909   [(set (match_operand:DI 0 "s_register_operand" "")
 910         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 911    (clobber (match_scratch:DI 2 ""))
 912    (clobber (reg:CC CC_REGNUM))]
 913   "TARGET_32BIT && reload_completed
 914    && arm_general_register_operand (operands[0], DImode)"
 915   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 916               (clobber (reg:CC CC_REGNUM))])]
 917   ""
 918 )
 919
 920 (define_insn "*umin<mode>3_neon"
 921   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 922         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 923                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 924   "TARGET_NEON"
 925   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 926   [(set_attr "neon_type" "neon_int_5")]
 927 )
 928
 929 (define_insn "*umax<mode>3_neon"
 930   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 931         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 932                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 933   "TARGET_NEON"
 934   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 935   [(set_attr "neon_type" "neon_int_5")]
 936 )
 937
 938 (define_insn "*smin<mode>3_neon"
 939   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 940         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 941                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 942   "TARGET_NEON"
 943   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 944   [(set (attr "neon_type")
 945       (if_then_else (match_test "<Is_float_mode>")
 946                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 947                     (const_string "neon_int_5")))]
 948 )
 949
 950 (define_insn "*smax<mode>3_neon"
 951   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 952         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 953                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 954   "TARGET_NEON"
 955   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 956   [(set (attr "neon_type")
 957       (if_then_else (match_test "<Is_float_mode>")
 958                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 959                     (const_string "neon_int_5")))]
 960 )
 961
 962 ; TODO: V2DI shifts are current disabled because there are bugs in the
 963 ; generic vectorizer code.  It ends up creating a V2DI constructor with
 964 ; SImode elements.
 965
 966 (define_insn "vashl<mode>3"
 967   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
 968         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
 969                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
 970   "TARGET_NEON"
 971   {
 972     switch (which_alternative)
 973       {
 974         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 975         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
 976                                                     <MODE>mode,
 977                                                     VALID_NEON_QREG_MODE (<MODE>mode),
 978                                                     true);
 979         default: gcc_unreachable ();
 980       }
 981   }
 982   [(set (attr "neon_type")
 983       (if_then_else (match_test "<Is_d_reg>")
 984                     (const_string "neon_vshl_ddd")
 985                     (const_string "neon_shift_3")))]
 986 )
 987
 988 (define_insn "vashr<mode>3_imm"
 989   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 990         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 991                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 992   "TARGET_NEON"
 993   {
 994     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 995                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 996                                         false);
 997   }
 998   [(set (attr "neon_type")
 999       (if_then_else (match_test "<Is_d_reg>")
1000                     (const_string "neon_vshl_ddd")
1001                     (const_string "neon_shift_3")))]
1002 )
1003
1004 (define_insn "vlshr<mode>3_imm"
1005   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1006         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1007                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1008   "TARGET_NEON"
1009   {
1010     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1011                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1012                                         false);
1013   }
1014   [(set (attr "neon_type")
1015         (if_then_else (match_test "<Is_d_reg>")
1016                       (const_string "neon_vshl_ddd")
1017                       (const_string "neon_shift_3")))]
1018 )
1019
1020 ; Used for implementing logical shift-right, which is a left-shift by a negative
1021 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1022 ; above, but using an unspec in case GCC tries anything tricky with negative
1023 ; shift amounts.
1024
1025 (define_insn "ashl<mode>3_signed"
1026   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1027         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1028                       (match_operand:VDQI 2 "s_register_operand" "w")]
1029                      UNSPEC_ASHIFT_SIGNED))]
1030   "TARGET_NEON"
1031   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1032   [(set (attr "neon_type")
1033       (if_then_else (match_test "<Is_d_reg>")
1034                     (const_string "neon_vshl_ddd")
1035                     (const_string "neon_shift_3")))]
1036 )
1037
1038 ; Used for implementing logical shift-right, which is a left-shift by a negative
1039 ; amount, with unsigned operands.
1040
1041 (define_insn "ashl<mode>3_unsigned"
1042   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1043         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1044                       (match_operand:VDQI 2 "s_register_operand" "w")]
1045                      UNSPEC_ASHIFT_UNSIGNED))]
1046   "TARGET_NEON"
1047   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1048   [(set (attr "neon_type")
1049       (if_then_else (match_test "<Is_d_reg>")
1050                     (const_string "neon_vshl_ddd")
1051                     (const_string "neon_shift_3")))]
1052 )
1053
1054 (define_expand "vashr<mode>3"
1055   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1056         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1057                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1058   "TARGET_NEON"
1059 {
1060   if (s_register_operand (operands[2], <MODE>mode))
1061     {
1062       rtx neg = gen_reg_rtx (<MODE>mode);
1063       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1064       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1065     }
1066   else
1067     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1068   DONE;
1069 })
1070
1071 (define_expand "vlshr<mode>3"
1072   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1073         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1074                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1075   "TARGET_NEON"
1076 {
1077   if (s_register_operand (operands[2], <MODE>mode))
1078     {
1079       rtx neg = gen_reg_rtx (<MODE>mode);
1080       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1081       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1082     }
1083   else
1084     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1085   DONE;
1086 })
1087
1088 ;; 64-bit shifts
1089
1090 ;; This pattern loads a 32-bit shift count into a 64-bit NEON register,
1091 ;; leaving the upper half uninitalized.  This is OK since the shift
1092 ;; instruction only looks at the low 8 bits anyway.  To avoid confusing
1093 ;; data flow analysis however, we pretend the full register is set
1094 ;; using an unspec.
1095 (define_insn "neon_load_count"
1096   [(set (match_operand:DI 0 "s_register_operand" "=w,w")
1097         (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")]
1098                    UNSPEC_LOAD_COUNT))]
1099   "TARGET_NEON"
1100   "@
1101    vld1.32\t{%P0[0]}, %A1
1102    vmov.32\t%P0[0], %1"
1103   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
1104 )
1105
1106 (define_insn "ashldi3_neon_noclobber"
1107   [(set (match_operand:DI 0 "s_register_operand"            "=w,w")
1108         (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w")
1109                    (match_operand:DI 2 "reg_or_int_operand" " i,w")))]
1110   "TARGET_NEON && reload_completed
1111    && (!CONST_INT_P (operands[2])
1112        || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))"
1113   "@
1114    vshl.u64\t%P0, %P1, %2
1115    vshl.u64\t%P0, %P1, %P2"
1116   [(set_attr "neon_type" "neon_vshl_ddd,neon_vshl_ddd")]
1117 )
1118
1119 (define_insn_and_split "ashldi3_neon"
1120   [(set (match_operand:DI 0 "s_register_operand"            "= w, w,?&r,?r, ?w,w")
1121         (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w")
1122                    (match_operand:SI 2 "general_operand"    "rUm, i,  r, i,rUm,i")))
1123    (clobber (match_scratch:SI 3                             "= X, X,?&r, X,  X,X"))
1124    (clobber (match_scratch:SI 4                             "= X, X,?&r, X,  X,X"))
1125    (clobber (match_scratch:DI 5                             "=&w, X,  X, X, &w,X"))
1126    (clobber (reg:CC_C CC_REGNUM))]
1127   "TARGET_NEON"
1128   "#"
1129   "TARGET_NEON && reload_completed"
1130   [(const_int 0)]
1131   "
1132   {
1133     if (IS_VFP_REGNUM (REGNO (operands[0])))
1134       {
1135         if (CONST_INT_P (operands[2]))
1136           {
1137             if (INTVAL (operands[2]) < 1)
1138               {
1139                 emit_insn (gen_movdi (operands[0], operands[1]));
1140                 DONE;
1141               }
1142             else if (INTVAL (operands[2]) > 63)
1143               operands[2] = gen_rtx_CONST_INT (VOIDmode, 63);
1144           }
1145         else
1146           {
1147             emit_insn (gen_neon_load_count (operands[5], operands[2]));
1148             operands[2] = operands[5];
1149           }
1150
1151         /* Ditch the unnecessary clobbers.  */
1152         emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1],
1153                                                operands[2]));
1154       }
1155     else
1156       {
1157         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1158           /* This clobbers CC.  */
1159           emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
1160         else
1161           arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
1162                                          operands[2], operands[3], operands[4]);
1163       }
1164     DONE;
1165   }"
1166   [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")
1167    (set_attr "opt" "*,*,speed,speed,*,*")]
1168 )
1169
1170 ; The shift amount needs to be negated for right-shifts
1171 (define_insn "signed_shift_di3_neon"
1172   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1173         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1174                     (match_operand:DI 2 "s_register_operand" " w")]
1175                    UNSPEC_ASHIFT_SIGNED))]
1176   "TARGET_NEON && reload_completed"
1177   "vshl.s64\t%P0, %P1, %P2"
1178   [(set_attr "neon_type" "neon_vshl_ddd")]
1179 )
1180
1181 ; The shift amount needs to be negated for right-shifts
1182 (define_insn "unsigned_shift_di3_neon"
1183   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1184         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1185                     (match_operand:DI 2 "s_register_operand" " w")]
1186                    UNSPEC_ASHIFT_UNSIGNED))]
1187   "TARGET_NEON && reload_completed"
1188   "vshl.u64\t%P0, %P1, %P2"
1189   [(set_attr "neon_type" "neon_vshl_ddd")]
1190 )
1191
1192 (define_insn "ashrdi3_neon_imm_noclobber"
1193   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1194         (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1195                      (match_operand:DI 2 "const_int_operand"  " i")))]
1196   "TARGET_NEON && reload_completed
1197    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1198   "vshr.s64\t%P0, %P1, %2"
1199   [(set_attr "neon_type" "neon_vshl_ddd")]
1200 )
1201
1202 (define_insn "lshrdi3_neon_imm_noclobber"
1203   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1204         (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1205                      (match_operand:DI 2 "const_int_operand"  " i")))]
1206   "TARGET_NEON && reload_completed
1207    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1208   "vshr.u64\t%P0, %P1, %2"
1209   [(set_attr "neon_type" "neon_vshl_ddd")]
1210 )
1211
1212 ;; ashrdi3_neon
1213 ;; lshrdi3_neon
1214 (define_insn_and_split "<shift>di3_neon"
1215   [(set (match_operand:DI 0 "s_register_operand"             "= w, w,?&r,?r,?w,?w")
1216         (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w")
1217                     (match_operand:SI 2 "reg_or_int_operand" "  r, i,  r, i, r, i")))
1218    (clobber (match_scratch:SI 3                              "=2r, X, &r, X,2r, X"))
1219    (clobber (match_scratch:SI 4                              "= X, X, &r, X, X, X"))
1220    (clobber (match_scratch:DI 5                              "=&w, X,  X, X,&w, X"))
1221    (clobber (reg:CC CC_REGNUM))]
1222   "TARGET_NEON"
1223   "#"
1224   "TARGET_NEON && reload_completed"
1225   [(const_int 0)]
1226   "
1227   {
1228     if (IS_VFP_REGNUM (REGNO (operands[0])))
1229       {
1230         if (CONST_INT_P (operands[2]))
1231           {
1232             if (INTVAL (operands[2]) < 1)
1233               {
1234                 emit_insn (gen_movdi (operands[0], operands[1]));
1235                 DONE;
1236               }
1237             else if (INTVAL (operands[2]) > 64)
1238               operands[2] = gen_rtx_CONST_INT (VOIDmode, 64);
1239
1240             /* Ditch the unnecessary clobbers.  */
1241             emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0],
1242                                                           operands[1],
1243                                                           operands[2]));
1244           }
1245         else
1246           {
1247             /* We must use a negative left-shift.  */
1248             emit_insn (gen_negsi2 (operands[3], operands[2]));
1249             emit_insn (gen_neon_load_count (operands[5], operands[3]));
1250             emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1],
1251                                                        operands[5]));
1252           }
1253       }
1254     else
1255       {
1256         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1257           /* This clobbers CC.  */
1258           emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
1259         else
1260           /* This clobbers CC (ASHIFTRT by register only).  */
1261           arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
1262                                          operands[2], operands[3], operands[4]);
1263       }
1264
1265     DONE;
1266   }"
1267   [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")
1268    (set_attr "opt" "*,*,speed,speed,*,*")]
1269 )
1270
1271 ;; Widening operations
1272
1273 (define_insn "widen_ssum<mode>3"
1274   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1275         (plus:<V_widen> (sign_extend:<V_widen>
1276                           (match_operand:VW 1 "s_register_operand" "%w"))
1277                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1278   "TARGET_NEON"
1279   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1280   [(set_attr "neon_type" "neon_int_3")]
1281 )
1282
1283 (define_insn "widen_usum<mode>3"
1284   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1285         (plus:<V_widen> (zero_extend:<V_widen>
1286                           (match_operand:VW 1 "s_register_operand" "%w"))
1287                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1288   "TARGET_NEON"
1289   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1290   [(set_attr "neon_type" "neon_int_3")]
1291 )
1292
1293 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1294 ;; shift-count granularity. That's good enough for the middle-end's current
1295 ;; needs.
1296
1297 ;; Note that it's not safe to perform such an operation in big-endian mode,
1298 ;; due to element-ordering issues.
1299
1300 (define_expand "vec_shr_<mode>"
1301   [(match_operand:VDQ 0 "s_register_operand" "")
1302    (match_operand:VDQ 1 "s_register_operand" "")
1303    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1304   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1305 {
1306   rtx zero_reg;
1307   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1308   const int width = GET_MODE_BITSIZE (<MODE>mode);
1309   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1310   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1311     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1312
1313   if (num_bits == width)
1314     {
1315       emit_move_insn (operands[0], operands[1]);
1316       DONE;
1317     }
1318
1319   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1320   operands[0] = gen_lowpart (bvecmode, operands[0]);
1321   operands[1] = gen_lowpart (bvecmode, operands[1]);
1322
1323   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1324                       GEN_INT (num_bits / BITS_PER_UNIT)));
1325   DONE;
1326 })
1327
1328 (define_expand "vec_shl_<mode>"
1329   [(match_operand:VDQ 0 "s_register_operand" "")
1330    (match_operand:VDQ 1 "s_register_operand" "")
1331    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1332   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1333 {
1334   rtx zero_reg;
1335   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1336   const int width = GET_MODE_BITSIZE (<MODE>mode);
1337   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1338   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1339     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1340
1341   if (num_bits == 0)
1342     {
1343       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1344       DONE;
1345     }
1346
1347   num_bits = width - num_bits;
1348
1349   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1350   operands[0] = gen_lowpart (bvecmode, operands[0]);
1351   operands[1] = gen_lowpart (bvecmode, operands[1]);
1352
1353   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1354                       GEN_INT (num_bits / BITS_PER_UNIT)));
1355   DONE;
1356 })
1357
1358 ;; Helpers for quad-word reduction operations
1359
1360 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1361 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1362 ; N/2-element vector.
1363
1364 (define_insn "quad_halves_<code>v4si"
1365   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1366         (vqh_ops:V2SI
1367           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1368                            (parallel [(const_int 0) (const_int 1)]))
1369           (vec_select:V2SI (match_dup 1)
1370                            (parallel [(const_int 2) (const_int 3)]))))]
1371   "TARGET_NEON"
1372   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1373   [(set_attr "vqh_mnem" "<VQH_mnem>")
1374    (set (attr "neon_type")
1375       (if_then_else (eq_attr "vqh_mnem" "vadd")
1376                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1377 )
1378
1379 (define_insn "quad_halves_<code>v4sf"
1380   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1381         (vqhs_ops:V2SF
1382           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1383                            (parallel [(const_int 0) (const_int 1)]))
1384           (vec_select:V2SF (match_dup 1)
1385                            (parallel [(const_int 2) (const_int 3)]))))]
1386   "TARGET_NEON && flag_unsafe_math_optimizations"
1387   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1388   [(set_attr "vqh_mnem" "<VQH_mnem>")
1389    (set (attr "neon_type")
1390       (if_then_else (eq_attr "vqh_mnem" "vadd")
1391                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1392 )
1393
1394 (define_insn "quad_halves_<code>v8hi"
1395   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1396         (vqh_ops:V4HI
1397           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1398                            (parallel [(const_int 0) (const_int 1)
1399                                       (const_int 2) (const_int 3)]))
1400           (vec_select:V4HI (match_dup 1)
1401                            (parallel [(const_int 4) (const_int 5)
1402                                       (const_int 6) (const_int 7)]))))]
1403   "TARGET_NEON"
1404   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1405   [(set_attr "vqh_mnem" "<VQH_mnem>")
1406    (set (attr "neon_type")
1407       (if_then_else (eq_attr "vqh_mnem" "vadd")
1408                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1409 )
1410
1411 (define_insn "quad_halves_<code>v16qi"
1412   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1413         (vqh_ops:V8QI
1414           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1415                            (parallel [(const_int 0) (const_int 1)
1416                                       (const_int 2) (const_int 3)
1417                                       (const_int 4) (const_int 5)
1418                                       (const_int 6) (const_int 7)]))
1419           (vec_select:V8QI (match_dup 1)
1420                            (parallel [(const_int 8) (const_int 9)
1421                                       (const_int 10) (const_int 11)
1422                                       (const_int 12) (const_int 13)
1423                                       (const_int 14) (const_int 15)]))))]
1424   "TARGET_NEON"
1425   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1426   [(set_attr "vqh_mnem" "<VQH_mnem>")
1427    (set (attr "neon_type")
1428       (if_then_else (eq_attr "vqh_mnem" "vadd")
1429                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1430 )
1431
1432 (define_expand "move_hi_quad_<mode>"
1433  [(match_operand:ANY128 0 "s_register_operand" "")
1434   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1435  "TARGET_NEON"
1436 {
1437   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1438                                        GET_MODE_SIZE (<V_HALF>mode)),
1439                   operands[1]);
1440   DONE;
1441 })
1442
1443 (define_expand "move_lo_quad_<mode>"
1444  [(match_operand:ANY128 0 "s_register_operand" "")
1445   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1446  "TARGET_NEON"
1447 {
1448   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1449                                        <MODE>mode, 0),
1450                   operands[1]);
1451   DONE;
1452 })
1453
1454 ;; Reduction operations
1455
1456 (define_expand "reduc_splus_<mode>"
1457   [(match_operand:VD 0 "s_register_operand" "")
1458    (match_operand:VD 1 "s_register_operand" "")]
1459   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1460 {
1461   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1462                         &gen_neon_vpadd_internal<mode>);
1463   DONE;
1464 })
1465
1466 (define_expand "reduc_splus_<mode>"
1467   [(match_operand:VQ 0 "s_register_operand" "")
1468    (match_operand:VQ 1 "s_register_operand" "")]
1469   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1470    && !BYTES_BIG_ENDIAN"
1471 {
1472   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1473   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1474
1475   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1476   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1477   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1478
1479   DONE;
1480 })
1481
1482 (define_insn "reduc_splus_v2di"
1483   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1484         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1485                      UNSPEC_VPADD))]
1486   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1487   "vadd.i64\t%e0, %e1, %f1"
1488   [(set_attr "neon_type" "neon_int_1")]
1489 )
1490
1491 ;; NEON does not distinguish between signed and unsigned addition except on
1492 ;; widening operations.
1493 (define_expand "reduc_uplus_<mode>"
1494   [(match_operand:VDQI 0 "s_register_operand" "")
1495    (match_operand:VDQI 1 "s_register_operand" "")]
1496   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1497 {
1498   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1499   DONE;
1500 })
1501
1502 (define_expand "reduc_smin_<mode>"
1503   [(match_operand:VD 0 "s_register_operand" "")
1504    (match_operand:VD 1 "s_register_operand" "")]
1505   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1506 {
1507   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1508                         &gen_neon_vpsmin<mode>);
1509   DONE;
1510 })
1511
1512 (define_expand "reduc_smin_<mode>"
1513   [(match_operand:VQ 0 "s_register_operand" "")
1514    (match_operand:VQ 1 "s_register_operand" "")]
1515   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1516    && !BYTES_BIG_ENDIAN"
1517 {
1518   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1519   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1520
1521   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1522   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1523   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1524
1525   DONE;
1526 })
1527
1528 (define_expand "reduc_smax_<mode>"
1529   [(match_operand:VD 0 "s_register_operand" "")
1530    (match_operand:VD 1 "s_register_operand" "")]
1531   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1532 {
1533   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1534                         &gen_neon_vpsmax<mode>);
1535   DONE;
1536 })
1537
1538 (define_expand "reduc_smax_<mode>"
1539   [(match_operand:VQ 0 "s_register_operand" "")
1540    (match_operand:VQ 1 "s_register_operand" "")]
1541   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1542    && !BYTES_BIG_ENDIAN"
1543 {
1544   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1545   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1546
1547   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1548   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1549   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1550
1551   DONE;
1552 })
1553
1554 (define_expand "reduc_umin_<mode>"
1555   [(match_operand:VDI 0 "s_register_operand" "")
1556    (match_operand:VDI 1 "s_register_operand" "")]
1557   "TARGET_NEON"
1558 {
1559   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1560                         &gen_neon_vpumin<mode>);
1561   DONE;
1562 })
1563
1564 (define_expand "reduc_umin_<mode>"
1565   [(match_operand:VQI 0 "s_register_operand" "")
1566    (match_operand:VQI 1 "s_register_operand" "")]
1567   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1568 {
1569   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1570   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1571
1572   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1573   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1574   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1575
1576   DONE;
1577 })
1578
1579 (define_expand "reduc_umax_<mode>"
1580   [(match_operand:VDI 0 "s_register_operand" "")
1581    (match_operand:VDI 1 "s_register_operand" "")]
1582   "TARGET_NEON"
1583 {
1584   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1585                         &gen_neon_vpumax<mode>);
1586   DONE;
1587 })
1588
1589 (define_expand "reduc_umax_<mode>"
1590   [(match_operand:VQI 0 "s_register_operand" "")
1591    (match_operand:VQI 1 "s_register_operand" "")]
1592   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1593 {
1594   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1595   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1596
1597   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1598   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1599   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1600
1601   DONE;
1602 })
1603
1604 (define_insn "neon_vpadd_internal<mode>"
1605   [(set (match_operand:VD 0 "s_register_operand" "=w")
1606         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1607                     (match_operand:VD 2 "s_register_operand" "w")]
1608                    UNSPEC_VPADD))]
1609   "TARGET_NEON"
1610   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1611   ;; Assume this schedules like vadd.
1612   [(set (attr "neon_type")
1613       (if_then_else (match_test "<Is_float_mode>")
1614                     (if_then_else (match_test "<Is_d_reg>")
1615                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1616                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1617                     (const_string "neon_int_1")))]
1618 )
1619
1620 (define_insn "neon_vpsmin<mode>"
1621   [(set (match_operand:VD 0 "s_register_operand" "=w")
1622         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1623                     (match_operand:VD 2 "s_register_operand" "w")]
1624                    UNSPEC_VPSMIN))]
1625   "TARGET_NEON"
1626   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1627   ;; Assume this schedules like vmin.
1628   [(set (attr "neon_type")
1629       (if_then_else (match_test "<Is_float_mode>")
1630                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1631                     (const_string "neon_int_5")))]
1632 )
1633
1634 (define_insn "neon_vpsmax<mode>"
1635   [(set (match_operand:VD 0 "s_register_operand" "=w")
1636         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1637                     (match_operand:VD 2 "s_register_operand" "w")]
1638                    UNSPEC_VPSMAX))]
1639   "TARGET_NEON"
1640   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1641   ;; Assume this schedules like vmax.
1642   [(set (attr "neon_type")
1643       (if_then_else (match_test "<Is_float_mode>")
1644                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1645                     (const_string "neon_int_5")))]
1646 )
1647
1648 (define_insn "neon_vpumin<mode>"
1649   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1650         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1651                      (match_operand:VDI 2 "s_register_operand" "w")]
1652                    UNSPEC_VPUMIN))]
1653   "TARGET_NEON"
1654   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1655   ;; Assume this schedules like umin.
1656   [(set_attr "neon_type" "neon_int_5")]
1657 )
1658
1659 (define_insn "neon_vpumax<mode>"
1660   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1661         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1662                      (match_operand:VDI 2 "s_register_operand" "w")]
1663                    UNSPEC_VPUMAX))]
1664   "TARGET_NEON"
1665   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1666   ;; Assume this schedules like umax.
1667   [(set_attr "neon_type" "neon_int_5")]
1668 )
1669
1670 ;; Saturating arithmetic
1671
1672 ; NOTE: Neon supports many more saturating variants of instructions than the
1673 ; following, but these are all GCC currently understands.
1674 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1675 ; yet either, although these patterns may be used by intrinsics when they're
1676 ; added.
1677
1678 (define_insn "*ss_add<mode>_neon"
1679   [(set (match_operand:VD 0 "s_register_operand" "=w")
1680        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1681                    (match_operand:VD 2 "s_register_operand" "w")))]
1682   "TARGET_NEON"
1683   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1684   [(set_attr "neon_type" "neon_int_4")]
1685 )
1686
1687 (define_insn "*us_add<mode>_neon"
1688   [(set (match_operand:VD 0 "s_register_operand" "=w")
1689        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1690                    (match_operand:VD 2 "s_register_operand" "w")))]
1691   "TARGET_NEON"
1692   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1693   [(set_attr "neon_type" "neon_int_4")]
1694 )
1695
1696 (define_insn "*ss_sub<mode>_neon"
1697   [(set (match_operand:VD 0 "s_register_operand" "=w")
1698        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1699                     (match_operand:VD 2 "s_register_operand" "w")))]
1700   "TARGET_NEON"
1701   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1702   [(set_attr "neon_type" "neon_int_5")]
1703 )
1704
1705 (define_insn "*us_sub<mode>_neon"
1706   [(set (match_operand:VD 0 "s_register_operand" "=w")
1707        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1708                     (match_operand:VD 2 "s_register_operand" "w")))]
1709   "TARGET_NEON"
1710   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1711   [(set_attr "neon_type" "neon_int_5")]
1712 )
1713
1714 ;; Conditional instructions.  These are comparisons with conditional moves for
1715 ;; vectors.  They perform the assignment:
1716 ;;
1717 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1718 ;;
1719 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1720 ;; element-wise.
1721
1722 (define_expand "vcond<mode><mode>"
1723   [(set (match_operand:VDQW 0 "s_register_operand" "")
1724         (if_then_else:VDQW
1725           (match_operator 3 "arm_comparison_operator"
1726             [(match_operand:VDQW 4 "s_register_operand" "")
1727              (match_operand:VDQW 5 "nonmemory_operand" "")])
1728           (match_operand:VDQW 1 "s_register_operand" "")
1729           (match_operand:VDQW 2 "s_register_operand" "")))]
1730   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1731 {
1732   rtx mask;
1733   int inverse = 0, immediate_zero = 0;
1734   /* See the description of "magic" bits in the 'T' case of
1735      arm_print_operand.  */
1736   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1737                              ? 3 : 1;
1738   rtx magic_rtx = GEN_INT (magic_word);
1739
1740   mask = gen_reg_rtx (<V_cmp_result>mode);
1741
1742   if (operands[5] == CONST0_RTX (<MODE>mode))
1743     immediate_zero = 1;
1744   else if (!REG_P (operands[5]))
1745     operands[5] = force_reg (<MODE>mode, operands[5]);
1746
1747   switch (GET_CODE (operands[3]))
1748     {
1749     case GE:
1750       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1751                                       magic_rtx));
1752       break;
1753
1754     case GT:
1755       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1756                                       magic_rtx));
1757       break;
1758
1759     case EQ:
1760       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1761                                       magic_rtx));
1762       break;
1763
1764     case LE:
1765       if (immediate_zero)
1766         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1767                                         magic_rtx));
1768       else
1769         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1770                                         magic_rtx));
1771       break;
1772
1773     case LT:
1774       if (immediate_zero)
1775         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1776                                         magic_rtx));
1777       else
1778         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1779                                         magic_rtx));
1780       break;
1781
1782     case NE:
1783       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1784                                       magic_rtx));
1785       inverse = 1;
1786       break;
1787
1788     default:
1789       gcc_unreachable ();
1790     }
1791
1792   if (inverse)
1793     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1794                                     operands[1]));
1795   else
1796     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1797                                     operands[2]));
1798
1799   DONE;
1800 })
1801
1802 (define_expand "vcondu<mode><mode>"
1803   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1804         (if_then_else:VDQIW
1805           (match_operator 3 "arm_comparison_operator"
1806             [(match_operand:VDQIW 4 "s_register_operand" "")
1807              (match_operand:VDQIW 5 "s_register_operand" "")])
1808           (match_operand:VDQIW 1 "s_register_operand" "")
1809           (match_operand:VDQIW 2 "s_register_operand" "")))]
1810   "TARGET_NEON"
1811 {
1812   rtx mask;
1813   int inverse = 0, immediate_zero = 0;
1814
1815   mask = gen_reg_rtx (<V_cmp_result>mode);
1816
1817   if (operands[5] == CONST0_RTX (<MODE>mode))
1818     immediate_zero = 1;
1819   else if (!REG_P (operands[5]))
1820     operands[5] = force_reg (<MODE>mode, operands[5]);
1821
1822   switch (GET_CODE (operands[3]))
1823     {
1824     case GEU:
1825       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1826                                       const0_rtx));
1827       break;
1828
1829     case GTU:
1830       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1831                                       const0_rtx));
1832       break;
1833
1834     case EQ:
1835       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1836                                       const0_rtx));
1837       break;
1838
1839     case LEU:
1840       if (immediate_zero)
1841         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1842                                         const0_rtx));
1843       else
1844         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1845                                         const0_rtx));
1846       break;
1847
1848     case LTU:
1849       if (immediate_zero)
1850         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1851                                         const0_rtx));
1852       else
1853         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1854                                         const0_rtx));
1855       break;
1856
1857     case NE:
1858       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1859                                       const0_rtx));
1860       inverse = 1;
1861       break;
1862
1863     default:
1864       gcc_unreachable ();
1865     }
1866
1867   if (inverse)
1868     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1869                                     operands[1]));
1870   else
1871     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1872                                     operands[2]));
1873
1874   DONE;
1875 })
1876
1877 ;; Patterns for builtins.
1878
1879 ; good for plain vadd, vaddq.
1880
1881 (define_expand "neon_vadd<mode>"
1882   [(match_operand:VDQX 0 "s_register_operand" "=w")
1883    (match_operand:VDQX 1 "s_register_operand" "w")
1884    (match_operand:VDQX 2 "s_register_operand" "w")
1885    (match_operand:SI 3 "immediate_operand" "i")]
1886   "TARGET_NEON"
1887 {
1888   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1889     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1890   else
1891     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1892                                            operands[2]));
1893   DONE;
1894 })
1895
1896 ; Note that NEON operations don't support the full IEEE 754 standard: in
1897 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1898 ; use those instructions for autovectorization, etc. unless
1899 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1900 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1901 ; header) must work in either case: if -funsafe-math-optimizations is given,
1902 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1903 ; expand to unspecs (which may potentially limit the extent to which they might
1904 ; be optimized by generic code).
1905
1906 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1907
1908 (define_insn "neon_vadd<mode>_unspec"
1909   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1910         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1911                       (match_operand:VDQX 2 "s_register_operand" "w")]
1912                      UNSPEC_VADD))]
1913   "TARGET_NEON"
1914   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1915   [(set (attr "neon_type")
1916       (if_then_else (match_test "<Is_float_mode>")
1917                     (if_then_else (match_test "<Is_d_reg>")
1918                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1919                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1920                     (const_string "neon_int_1")))]
1921 )
1922
1923 ; operand 3 represents in bits:
1924 ;  bit 0: signed (vs unsigned).
1925 ;  bit 1: rounding (vs none).
1926
1927 (define_insn "neon_vaddl<mode>"
1928   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1929         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1930                            (match_operand:VDI 2 "s_register_operand" "w")
1931                            (match_operand:SI 3 "immediate_operand" "i")]
1932                           UNSPEC_VADDL))]
1933   "TARGET_NEON"
1934   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1935   [(set_attr "neon_type" "neon_int_3")]
1936 )
1937
1938 (define_insn "neon_vaddw<mode>"
1939   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1940         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1941                            (match_operand:VDI 2 "s_register_operand" "w")
1942                            (match_operand:SI 3 "immediate_operand" "i")]
1943                           UNSPEC_VADDW))]
1944   "TARGET_NEON"
1945   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1946   [(set_attr "neon_type" "neon_int_2")]
1947 )
1948
1949 ; vhadd and vrhadd.
1950
1951 (define_insn "neon_vhadd<mode>"
1952   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1953         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1954                        (match_operand:VDQIW 2 "s_register_operand" "w")
1955                        (match_operand:SI 3 "immediate_operand" "i")]
1956                       UNSPEC_VHADD))]
1957   "TARGET_NEON"
1958   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1959   [(set_attr "neon_type" "neon_int_4")]
1960 )
1961
1962 (define_insn "neon_vqadd<mode>"
1963   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1964         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1965                        (match_operand:VDQIX 2 "s_register_operand" "w")
1966                        (match_operand:SI 3 "immediate_operand" "i")]
1967                      UNSPEC_VQADD))]
1968   "TARGET_NEON"
1969   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1970   [(set_attr "neon_type" "neon_int_4")]
1971 )
1972
1973 (define_insn "neon_vaddhn<mode>"
1974   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1975         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1976                             (match_operand:VN 2 "s_register_operand" "w")
1977                             (match_operand:SI 3 "immediate_operand" "i")]
1978                            UNSPEC_VADDHN))]
1979   "TARGET_NEON"
1980   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1981   [(set_attr "neon_type" "neon_int_4")]
1982 )
1983
1984 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1985 ;; polynomial multiplication case that can specified by operand 3.
1986 (define_insn "neon_vmul<mode>"
1987   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1988         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1989                       (match_operand:VDQW 2 "s_register_operand" "w")
1990                       (match_operand:SI 3 "immediate_operand" "i")]
1991                      UNSPEC_VMUL))]
1992   "TARGET_NEON"
1993   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1994   [(set (attr "neon_type")
1995       (if_then_else (match_test "<Is_float_mode>")
1996                     (if_then_else (match_test "<Is_d_reg>")
1997                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1998                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1999                     (if_then_else (match_test "<Is_d_reg>")
2000                                   (if_then_else
2001                                     (match_test "<Scalar_mul_8_16>")
2002                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2003                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2004                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2005                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
2006                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
2007 )
2008
2009 (define_expand "neon_vmla<mode>"
2010   [(match_operand:VDQW 0 "s_register_operand" "=w")
2011    (match_operand:VDQW 1 "s_register_operand" "0")
2012    (match_operand:VDQW 2 "s_register_operand" "w")
2013    (match_operand:VDQW 3 "s_register_operand" "w")
2014    (match_operand:SI 4 "immediate_operand" "i")]
2015   "TARGET_NEON"
2016 {
2017   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2018     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
2019                                              operands[2], operands[3]));
2020   else
2021     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
2022                                            operands[2], operands[3]));
2023   DONE;
2024 })
2025
2026 (define_expand "neon_vfma<VCVTF:mode>"
2027   [(match_operand:VCVTF 0 "s_register_operand")
2028    (match_operand:VCVTF 1 "s_register_operand")
2029    (match_operand:VCVTF 2 "s_register_operand")
2030    (match_operand:VCVTF 3 "s_register_operand")
2031    (match_operand:SI 4 "immediate_operand")]
2032   "TARGET_NEON && TARGET_FMA"
2033 {
2034   emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3],
2035                                        operands[1]));
2036   DONE;
2037 })
2038
2039 (define_expand "neon_vfms<VCVTF:mode>"
2040   [(match_operand:VCVTF 0 "s_register_operand")
2041    (match_operand:VCVTF 1 "s_register_operand")
2042    (match_operand:VCVTF 2 "s_register_operand")
2043    (match_operand:VCVTF 3 "s_register_operand")
2044    (match_operand:SI 4 "immediate_operand")]
2045   "TARGET_NEON && TARGET_FMA"
2046 {
2047   emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3],
2048                                          operands[1]));
2049   DONE;
2050 })
2051
2052 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2053
2054 (define_insn "neon_vmla<mode>_unspec"
2055   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2056         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2057                      (match_operand:VDQ 2 "s_register_operand" "w")
2058                      (match_operand:VDQ 3 "s_register_operand" "w")]
2059                     UNSPEC_VMLA))]
2060   "TARGET_NEON"
2061   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2062   [(set (attr "neon_type")
2063       (if_then_else (match_test "<Is_float_mode>")
2064                     (if_then_else (match_test "<Is_d_reg>")
2065                                   (const_string "neon_fp_vmla_ddd")
2066                                   (const_string "neon_fp_vmla_qqq"))
2067                     (if_then_else (match_test "<Is_d_reg>")
2068                                   (if_then_else
2069                                     (match_test "<Scalar_mul_8_16>")
2070                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2071                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2072                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2073                                     (const_string "neon_mla_qqq_8_16")
2074                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2075 )
2076
2077 (define_insn "neon_vmlal<mode>"
2078   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2079         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2080                            (match_operand:VW 2 "s_register_operand" "w")
2081                            (match_operand:VW 3 "s_register_operand" "w")
2082                            (match_operand:SI 4 "immediate_operand" "i")]
2083                           UNSPEC_VMLAL))]
2084   "TARGET_NEON"
2085   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2086   [(set (attr "neon_type")
2087      (if_then_else (match_test "<Scalar_mul_8_16>")
2088                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2089                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2090 )
2091
2092 (define_expand "neon_vmls<mode>"
2093   [(match_operand:VDQW 0 "s_register_operand" "=w")
2094    (match_operand:VDQW 1 "s_register_operand" "0")
2095    (match_operand:VDQW 2 "s_register_operand" "w")
2096    (match_operand:VDQW 3 "s_register_operand" "w")
2097    (match_operand:SI 4 "immediate_operand" "i")]
2098   "TARGET_NEON"
2099 {
2100   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2101     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
2102                  operands[1], operands[2], operands[3]));
2103   else
2104     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
2105                                            operands[2], operands[3]));
2106   DONE;
2107 })
2108
2109 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2110
2111 (define_insn "neon_vmls<mode>_unspec"
2112   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2113         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2114                      (match_operand:VDQ 2 "s_register_operand" "w")
2115                      (match_operand:VDQ 3 "s_register_operand" "w")]
2116                     UNSPEC_VMLS))]
2117   "TARGET_NEON"
2118   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2119   [(set (attr "neon_type")
2120       (if_then_else (match_test "<Is_float_mode>")
2121                     (if_then_else (match_test "<Is_d_reg>")
2122                                   (const_string "neon_fp_vmla_ddd")
2123                                   (const_string "neon_fp_vmla_qqq"))
2124                     (if_then_else (match_test "<Is_d_reg>")
2125                                   (if_then_else
2126                                     (match_test "<Scalar_mul_8_16>")
2127                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2128                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2129                                   (if_then_else
2130                                     (match_test "<Scalar_mul_8_16>")
2131                                     (const_string "neon_mla_qqq_8_16")
2132                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2133 )
2134
2135 (define_insn "neon_vmlsl<mode>"
2136   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2137         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2138                            (match_operand:VW 2 "s_register_operand" "w")
2139                            (match_operand:VW 3 "s_register_operand" "w")
2140                            (match_operand:SI 4 "immediate_operand" "i")]
2141                           UNSPEC_VMLSL))]
2142   "TARGET_NEON"
2143   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2144   [(set (attr "neon_type")
2145      (if_then_else (match_test "<Scalar_mul_8_16>")
2146                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2147                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2148 )
2149
2150 (define_insn "neon_vqdmulh<mode>"
2151   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2152         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2153                        (match_operand:VMDQI 2 "s_register_operand" "w")
2154                        (match_operand:SI 3 "immediate_operand" "i")]
2155                       UNSPEC_VQDMULH))]
2156   "TARGET_NEON"
2157   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2158   [(set (attr "neon_type")
2159      (if_then_else (match_test "<Is_d_reg>")
2160         (if_then_else (match_test "<Scalar_mul_8_16>")
2161                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2162                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2163         (if_then_else (match_test "<Scalar_mul_8_16>")
2164                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2165                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2166 )
2167
2168 (define_insn "neon_vqdmlal<mode>"
2169   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2170         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2171                            (match_operand:VMDI 2 "s_register_operand" "w")
2172                            (match_operand:VMDI 3 "s_register_operand" "w")
2173                            (match_operand:SI 4 "immediate_operand" "i")]
2174                           UNSPEC_VQDMLAL))]
2175   "TARGET_NEON"
2176   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2177   [(set (attr "neon_type")
2178      (if_then_else (match_test "<Scalar_mul_8_16>")
2179                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2180                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2181 )
2182
2183 (define_insn "neon_vqdmlsl<mode>"
2184   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2185         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2186                            (match_operand:VMDI 2 "s_register_operand" "w")
2187                            (match_operand:VMDI 3 "s_register_operand" "w")
2188                            (match_operand:SI 4 "immediate_operand" "i")]
2189                           UNSPEC_VQDMLSL))]
2190   "TARGET_NEON"
2191   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2192   [(set (attr "neon_type")
2193      (if_then_else (match_test "<Scalar_mul_8_16>")
2194                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2195                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2196 )
2197
2198 (define_insn "neon_vmull<mode>"
2199   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2200         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2201                            (match_operand:VW 2 "s_register_operand" "w")
2202                            (match_operand:SI 3 "immediate_operand" "i")]
2203                           UNSPEC_VMULL))]
2204   "TARGET_NEON"
2205   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2206   [(set (attr "neon_type")
2207      (if_then_else (match_test "<Scalar_mul_8_16>")
2208                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2209                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2210 )
2211
2212 (define_insn "neon_vqdmull<mode>"
2213   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2214         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2215                            (match_operand:VMDI 2 "s_register_operand" "w")
2216                            (match_operand:SI 3 "immediate_operand" "i")]
2217                           UNSPEC_VQDMULL))]
2218   "TARGET_NEON"
2219   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2220   [(set (attr "neon_type")
2221      (if_then_else (match_test "<Scalar_mul_8_16>")
2222                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2223                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2224 )
2225
2226 (define_expand "neon_vsub<mode>"
2227   [(match_operand:VDQX 0 "s_register_operand" "=w")
2228    (match_operand:VDQX 1 "s_register_operand" "w")
2229    (match_operand:VDQX 2 "s_register_operand" "w")
2230    (match_operand:SI 3 "immediate_operand" "i")]
2231   "TARGET_NEON"
2232 {
2233   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2234     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2235   else
2236     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2237                                            operands[2]));
2238   DONE;
2239 })
2240
2241 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2242
2243 (define_insn "neon_vsub<mode>_unspec"
2244   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2245         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2246                       (match_operand:VDQX 2 "s_register_operand" "w")]
2247                      UNSPEC_VSUB))]
2248   "TARGET_NEON"
2249   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2250   [(set (attr "neon_type")
2251       (if_then_else (match_test "<Is_float_mode>")
2252                     (if_then_else (match_test "<Is_d_reg>")
2253                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2254                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2255                     (const_string "neon_int_2")))]
2256 )
2257
2258 (define_insn "neon_vsubl<mode>"
2259   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2260         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2261                            (match_operand:VDI 2 "s_register_operand" "w")
2262                            (match_operand:SI 3 "immediate_operand" "i")]
2263                           UNSPEC_VSUBL))]
2264   "TARGET_NEON"
2265   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2266   [(set_attr "neon_type" "neon_int_2")]
2267 )
2268
2269 (define_insn "neon_vsubw<mode>"
2270   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2271         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2272                            (match_operand:VDI 2 "s_register_operand" "w")
2273                            (match_operand:SI 3 "immediate_operand" "i")]
2274                           UNSPEC_VSUBW))]
2275   "TARGET_NEON"
2276   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2277   [(set_attr "neon_type" "neon_int_2")]
2278 )
2279
2280 (define_insn "neon_vqsub<mode>"
2281   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2282         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2283                        (match_operand:VDQIX 2 "s_register_operand" "w")
2284                        (match_operand:SI 3 "immediate_operand" "i")]
2285                       UNSPEC_VQSUB))]
2286   "TARGET_NEON"
2287   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2288   [(set_attr "neon_type" "neon_int_5")]
2289 )
2290
2291 (define_insn "neon_vhsub<mode>"
2292   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2293         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2294                        (match_operand:VDQIW 2 "s_register_operand" "w")
2295                        (match_operand:SI 3 "immediate_operand" "i")]
2296                       UNSPEC_VHSUB))]
2297   "TARGET_NEON"
2298   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2299   [(set_attr "neon_type" "neon_int_5")]
2300 )
2301
2302 (define_insn "neon_vsubhn<mode>"
2303   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2304         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2305                             (match_operand:VN 2 "s_register_operand" "w")
2306                             (match_operand:SI 3 "immediate_operand" "i")]
2307                            UNSPEC_VSUBHN))]
2308   "TARGET_NEON"
2309   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2310   [(set_attr "neon_type" "neon_int_4")]
2311 )
2312
2313 (define_insn "neon_vceq<mode>"
2314   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2315         (unspec:<V_cmp_result>
2316           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2317            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2318            (match_operand:SI 3 "immediate_operand" "i,i")]
2319           UNSPEC_VCEQ))]
2320   "TARGET_NEON"
2321   "@
2322   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2323   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2324   [(set (attr "neon_type")
2325       (if_then_else (match_test "<Is_float_mode>")
2326                     (if_then_else (match_test "<Is_d_reg>")
2327                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2328                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2329                     (const_string "neon_int_5")))]
2330 )
2331
2332 (define_insn "neon_vcge<mode>"
2333   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2334         (unspec:<V_cmp_result>
2335           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2336            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2337            (match_operand:SI 3 "immediate_operand" "i,i")]
2338           UNSPEC_VCGE))]
2339   "TARGET_NEON"
2340   "@
2341   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2342   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2343   [(set (attr "neon_type")
2344      (if_then_else (match_test "<Is_float_mode>")
2345                    (if_then_else (match_test "<Is_d_reg>")
2346                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2347                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2348                    (const_string "neon_int_5")))]
2349 )
2350
2351 (define_insn "neon_vcgeu<mode>"
2352   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2353         (unspec:<V_cmp_result>
2354           [(match_operand:VDQIW 1 "s_register_operand" "w")
2355            (match_operand:VDQIW 2 "s_register_operand" "w")
2356            (match_operand:SI 3 "immediate_operand" "i")]
2357           UNSPEC_VCGEU))]
2358   "TARGET_NEON"
2359   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2360   [(set_attr "neon_type" "neon_int_5")]
2361 )
2362
2363 (define_insn "neon_vcgt<mode>"
2364   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2365         (unspec:<V_cmp_result>
2366           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2367            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2368            (match_operand:SI 3 "immediate_operand" "i,i")]
2369           UNSPEC_VCGT))]
2370   "TARGET_NEON"
2371   "@
2372   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2373   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2374   [(set (attr "neon_type")
2375      (if_then_else (match_test "<Is_float_mode>")
2376                    (if_then_else (match_test "<Is_d_reg>")
2377                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2378                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2379                    (const_string "neon_int_5")))]
2380 )
2381
2382 (define_insn "neon_vcgtu<mode>"
2383   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2384         (unspec:<V_cmp_result>
2385           [(match_operand:VDQIW 1 "s_register_operand" "w")
2386            (match_operand:VDQIW 2 "s_register_operand" "w")
2387            (match_operand:SI 3 "immediate_operand" "i")]
2388           UNSPEC_VCGTU))]
2389   "TARGET_NEON"
2390   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2391   [(set_attr "neon_type" "neon_int_5")]
2392 )
2393
2394 ;; VCLE and VCLT only support comparisons with immediate zero (register
2395 ;; variants are VCGE and VCGT with operands reversed).
2396
2397 (define_insn "neon_vcle<mode>"
2398   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2399         (unspec:<V_cmp_result>
2400           [(match_operand:VDQW 1 "s_register_operand" "w")
2401            (match_operand:VDQW 2 "zero_operand" "Dz")
2402            (match_operand:SI 3 "immediate_operand" "i")]
2403           UNSPEC_VCLE))]
2404   "TARGET_NEON"
2405   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2406   [(set (attr "neon_type")
2407       (if_then_else (match_test "<Is_float_mode>")
2408                     (if_then_else (match_test "<Is_d_reg>")
2409                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2410                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2411                     (const_string "neon_int_5")))]
2412 )
2413
2414 (define_insn "neon_vclt<mode>"
2415   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2416         (unspec:<V_cmp_result>
2417           [(match_operand:VDQW 1 "s_register_operand" "w")
2418            (match_operand:VDQW 2 "zero_operand" "Dz")
2419            (match_operand:SI 3 "immediate_operand" "i")]
2420           UNSPEC_VCLT))]
2421   "TARGET_NEON"
2422   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2423   [(set (attr "neon_type")
2424       (if_then_else (match_test "<Is_float_mode>")
2425                     (if_then_else (match_test "<Is_d_reg>")
2426                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2427                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2428                     (const_string "neon_int_5")))]
2429 )
2430
2431 (define_insn "neon_vcage<mode>"
2432   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2433         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2434                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2435                                 (match_operand:SI 3 "immediate_operand" "i")]
2436                                UNSPEC_VCAGE))]
2437   "TARGET_NEON"
2438   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2439   [(set (attr "neon_type")
2440      (if_then_else (match_test "<Is_d_reg>")
2441                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2442                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2443 )
2444
2445 (define_insn "neon_vcagt<mode>"
2446   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2447         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2448                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2449                                 (match_operand:SI 3 "immediate_operand" "i")]
2450                                UNSPEC_VCAGT))]
2451   "TARGET_NEON"
2452   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2453   [(set (attr "neon_type")
2454      (if_then_else (match_test "<Is_d_reg>")
2455                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2456                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2457 )
2458
2459 (define_insn "neon_vtst<mode>"
2460   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2461         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2462                        (match_operand:VDQIW 2 "s_register_operand" "w")
2463                        (match_operand:SI 3 "immediate_operand" "i")]
2464                       UNSPEC_VTST))]
2465   "TARGET_NEON"
2466   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2467   [(set_attr "neon_type" "neon_int_4")]
2468 )
2469
2470 (define_insn "neon_vabd<mode>"
2471   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2472         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2473                       (match_operand:VDQW 2 "s_register_operand" "w")
2474                       (match_operand:SI 3 "immediate_operand" "i")]
2475                      UNSPEC_VABD))]
2476   "TARGET_NEON"
2477   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2478   [(set (attr "neon_type")
2479      (if_then_else (match_test "<Is_float_mode>")
2480                    (if_then_else (match_test "<Is_d_reg>")
2481                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2482                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2483                    (const_string "neon_int_5")))]
2484 )
2485
2486 (define_insn "neon_vabdl<mode>"
2487   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2488         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2489                            (match_operand:VW 2 "s_register_operand" "w")
2490                            (match_operand:SI 3 "immediate_operand" "i")]
2491                           UNSPEC_VABDL))]
2492   "TARGET_NEON"
2493   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2494   [(set_attr "neon_type" "neon_int_5")]
2495 )
2496
2497 (define_insn "neon_vaba<mode>"
2498   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2499         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2500                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2501                                    (match_operand:SI 4 "immediate_operand" "i")]
2502                                   UNSPEC_VABD)
2503                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2504   "TARGET_NEON"
2505   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2506   [(set (attr "neon_type")
2507      (if_then_else (match_test "<Is_d_reg>")
2508                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2509 )
2510
2511 (define_insn "neon_vabal<mode>"
2512   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2513         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2514                                            (match_operand:VW 3 "s_register_operand" "w")
2515                                            (match_operand:SI 4 "immediate_operand" "i")]
2516                                            UNSPEC_VABDL)
2517                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2518   "TARGET_NEON"
2519   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2520   [(set_attr "neon_type" "neon_vaba")]
2521 )
2522
2523 (define_insn "neon_vmax<mode>"
2524   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2525         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2526                       (match_operand:VDQW 2 "s_register_operand" "w")
2527                       (match_operand:SI 3 "immediate_operand" "i")]
2528                      UNSPEC_VMAX))]
2529   "TARGET_NEON"
2530   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2531   [(set (attr "neon_type")
2532     (if_then_else (match_test "<Is_float_mode>")
2533                   (if_then_else (match_test "<Is_d_reg>")
2534                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2535                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2536                   (const_string "neon_int_5")))]
2537 )
2538
2539 (define_insn "neon_vmin<mode>"
2540   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2541         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2542                       (match_operand:VDQW 2 "s_register_operand" "w")
2543                       (match_operand:SI 3 "immediate_operand" "i")]
2544                      UNSPEC_VMIN))]
2545   "TARGET_NEON"
2546   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2547   [(set (attr "neon_type")
2548     (if_then_else (match_test "<Is_float_mode>")
2549                   (if_then_else (match_test "<Is_d_reg>")
2550                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2551                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2552                   (const_string "neon_int_5")))]
2553 )
2554
2555 (define_expand "neon_vpadd<mode>"
2556   [(match_operand:VD 0 "s_register_operand" "=w")
2557    (match_operand:VD 1 "s_register_operand" "w")
2558    (match_operand:VD 2 "s_register_operand" "w")
2559    (match_operand:SI 3 "immediate_operand" "i")]
2560   "TARGET_NEON"
2561 {
2562   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2563                                             operands[2]));
2564   DONE;
2565 })
2566
2567 (define_insn "neon_vpaddl<mode>"
2568   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2569         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2570                                   (match_operand:SI 2 "immediate_operand" "i")]
2571                                  UNSPEC_VPADDL))]
2572   "TARGET_NEON"
2573   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2574   ;; Assume this schedules like vaddl.
2575   [(set_attr "neon_type" "neon_int_3")]
2576 )
2577
2578 (define_insn "neon_vpadal<mode>"
2579   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2580         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2581                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2582                                   (match_operand:SI 3 "immediate_operand" "i")]
2583                                  UNSPEC_VPADAL))]
2584   "TARGET_NEON"
2585   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2586   ;; Assume this schedules like vpadd.
2587   [(set_attr "neon_type" "neon_int_1")]
2588 )
2589
2590 (define_insn "neon_vpmax<mode>"
2591   [(set (match_operand:VD 0 "s_register_operand" "=w")
2592         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2593                     (match_operand:VD 2 "s_register_operand" "w")
2594                     (match_operand:SI 3 "immediate_operand" "i")]
2595                    UNSPEC_VPMAX))]
2596   "TARGET_NEON"
2597   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2598   ;; Assume this schedules like vmax.
2599   [(set (attr "neon_type")
2600     (if_then_else (match_test "<Is_float_mode>")
2601                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2602                   (const_string "neon_int_5")))]
2603 )
2604
2605 (define_insn "neon_vpmin<mode>"
2606   [(set (match_operand:VD 0 "s_register_operand" "=w")
2607         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2608                     (match_operand:VD 2 "s_register_operand" "w")
2609                     (match_operand:SI 3 "immediate_operand" "i")]
2610                    UNSPEC_VPMIN))]
2611   "TARGET_NEON"
2612   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2613   ;; Assume this schedules like vmin.
2614   [(set (attr "neon_type")
2615     (if_then_else (match_test "<Is_float_mode>")
2616                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2617                   (const_string "neon_int_5")))]
2618 )
2619
2620 (define_insn "neon_vrecps<mode>"
2621   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2622         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2623                        (match_operand:VCVTF 2 "s_register_operand" "w")
2624                        (match_operand:SI 3 "immediate_operand" "i")]
2625                       UNSPEC_VRECPS))]
2626   "TARGET_NEON"
2627   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2628   [(set (attr "neon_type")
2629       (if_then_else (match_test "<Is_d_reg>")
2630                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2631                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2632 )
2633
2634 (define_insn "neon_vrsqrts<mode>"
2635   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2636         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2637                        (match_operand:VCVTF 2 "s_register_operand" "w")
2638                        (match_operand:SI 3 "immediate_operand" "i")]
2639                       UNSPEC_VRSQRTS))]
2640   "TARGET_NEON"
2641   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2642   [(set (attr "neon_type")
2643       (if_then_else (match_test "<Is_d_reg>")
2644                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2645                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2646 )
2647
2648 (define_expand "neon_vabs<mode>"
2649   [(match_operand:VDQW 0 "s_register_operand" "")
2650    (match_operand:VDQW 1 "s_register_operand" "")
2651    (match_operand:SI 2 "immediate_operand" "")]
2652   "TARGET_NEON"
2653 {
2654   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2655   DONE;
2656 })
2657
2658 (define_insn "neon_vqabs<mode>"
2659   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2660         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2661                        (match_operand:SI 2 "immediate_operand" "i")]
2662                       UNSPEC_VQABS))]
2663   "TARGET_NEON"
2664   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2665   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2666 )
2667
2668 (define_expand "neon_vneg<mode>"
2669   [(match_operand:VDQW 0 "s_register_operand" "")
2670    (match_operand:VDQW 1 "s_register_operand" "")
2671    (match_operand:SI 2 "immediate_operand" "")]
2672   "TARGET_NEON"
2673 {
2674   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2675   DONE;
2676 })
2677
2678 (define_insn "neon_vqneg<mode>"
2679   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2680         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2681                        (match_operand:SI 2 "immediate_operand" "i")]
2682                       UNSPEC_VQNEG))]
2683   "TARGET_NEON"
2684   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2685   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2686 )
2687
2688 (define_insn "neon_vcls<mode>"
2689   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2690         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2691                        (match_operand:SI 2 "immediate_operand" "i")]
2692                       UNSPEC_VCLS))]
2693   "TARGET_NEON"
2694   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2695   [(set_attr "neon_type" "neon_int_1")]
2696 )
2697
2698 (define_insn "clz<mode>2"
2699   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2700         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2701   "TARGET_NEON"
2702   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2703   [(set_attr "neon_type" "neon_int_1")]
2704 )
2705
2706 (define_expand "neon_vclz<mode>"
2707   [(match_operand:VDQIW 0 "s_register_operand" "")
2708    (match_operand:VDQIW 1 "s_register_operand" "")
2709    (match_operand:SI 2 "immediate_operand" "")]
2710   "TARGET_NEON"
2711 {
2712   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2713   DONE;
2714 })
2715
2716 (define_insn "popcount<mode>2"
2717   [(set (match_operand:VE 0 "s_register_operand" "=w")
2718         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2719   "TARGET_NEON"
2720   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2721   [(set_attr "neon_type" "neon_int_1")]
2722 )
2723
2724 (define_expand "neon_vcnt<mode>"
2725   [(match_operand:VE 0 "s_register_operand" "=w")
2726    (match_operand:VE 1 "s_register_operand" "w")
2727    (match_operand:SI 2 "immediate_operand" "i")]
2728   "TARGET_NEON"
2729 {
2730   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2731   DONE;
2732 })
2733
2734 (define_insn "neon_vrecpe<mode>"
2735   [(set (match_operand:V32 0 "s_register_operand" "=w")
2736         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2737                      (match_operand:SI 2 "immediate_operand" "i")]
2738                     UNSPEC_VRECPE))]
2739   "TARGET_NEON"
2740   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2741   [(set (attr "neon_type")
2742       (if_then_else (match_test "<Is_d_reg>")
2743                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2744                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2745 )
2746
2747 (define_insn "neon_vrsqrte<mode>"
2748   [(set (match_operand:V32 0 "s_register_operand" "=w")
2749         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2750                      (match_operand:SI 2 "immediate_operand" "i")]
2751                     UNSPEC_VRSQRTE))]
2752   "TARGET_NEON"
2753   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2754   [(set (attr "neon_type")
2755       (if_then_else (match_test "<Is_d_reg>")
2756                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2757                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2758 )
2759
2760 (define_expand "neon_vmvn<mode>"
2761   [(match_operand:VDQIW 0 "s_register_operand" "")
2762    (match_operand:VDQIW 1 "s_register_operand" "")
2763    (match_operand:SI 2 "immediate_operand" "")]
2764   "TARGET_NEON"
2765 {
2766   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2767   DONE;
2768 })
2769
2770 (define_insn "neon_vget_lane<mode>_sext_internal"
2771   [(set (match_operand:SI 0 "s_register_operand" "=r")
2772         (sign_extend:SI
2773           (vec_select:<V_elem>
2774             (match_operand:VD 1 "s_register_operand" "w")
2775             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2776   "TARGET_NEON"
2777 {
2778   if (BYTES_BIG_ENDIAN)
2779     {
2780       int elt = INTVAL (operands[2]);
2781       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2782       operands[2] = GEN_INT (elt);
2783     }
2784   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2785 }
2786   [(set_attr "neon_type" "neon_bp_simple")]
2787 )
2788
2789 (define_insn "neon_vget_lane<mode>_zext_internal"
2790   [(set (match_operand:SI 0 "s_register_operand" "=r")
2791         (zero_extend:SI
2792           (vec_select:<V_elem>
2793             (match_operand:VD 1 "s_register_operand" "w")
2794             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2795   "TARGET_NEON"
2796 {
2797   if (BYTES_BIG_ENDIAN)
2798     {
2799       int elt = INTVAL (operands[2]);
2800       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2801       operands[2] = GEN_INT (elt);
2802     }
2803   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2804 }
2805   [(set_attr "neon_type" "neon_bp_simple")]
2806 )
2807
2808 (define_insn "neon_vget_lane<mode>_sext_internal"
2809   [(set (match_operand:SI 0 "s_register_operand" "=r")
2810         (sign_extend:SI
2811           (vec_select:<V_elem>
2812             (match_operand:VQ 1 "s_register_operand" "w")
2813             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2814   "TARGET_NEON"
2815 {
2816   rtx ops[3];
2817   int regno = REGNO (operands[1]);
2818   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2819   unsigned int elt = INTVAL (operands[2]);
2820   unsigned int elt_adj = elt % halfelts;
2821
2822   if (BYTES_BIG_ENDIAN)
2823     elt_adj = halfelts - 1 - elt_adj;
2824
2825   ops[0] = operands[0];
2826   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2827   ops[2] = GEN_INT (elt_adj);
2828   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2829
2830   return "";
2831 }
2832   [(set_attr "neon_type" "neon_bp_simple")]
2833 )
2834
2835 (define_insn "neon_vget_lane<mode>_zext_internal"
2836   [(set (match_operand:SI 0 "s_register_operand" "=r")
2837         (zero_extend:SI
2838           (vec_select:<V_elem>
2839             (match_operand:VQ 1 "s_register_operand" "w")
2840             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2841   "TARGET_NEON"
2842 {
2843   rtx ops[3];
2844   int regno = REGNO (operands[1]);
2845   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2846   unsigned int elt = INTVAL (operands[2]);
2847   unsigned int elt_adj = elt % halfelts;
2848
2849   if (BYTES_BIG_ENDIAN)
2850     elt_adj = halfelts - 1 - elt_adj;
2851
2852   ops[0] = operands[0];
2853   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2854   ops[2] = GEN_INT (elt_adj);
2855   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2856
2857   return "";
2858 }
2859   [(set_attr "neon_type" "neon_bp_simple")]
2860 )
2861
2862 (define_expand "neon_vget_lane<mode>"
2863   [(match_operand:<V_ext> 0 "s_register_operand" "")
2864    (match_operand:VDQW 1 "s_register_operand" "")
2865    (match_operand:SI 2 "immediate_operand" "")
2866    (match_operand:SI 3 "immediate_operand" "")]
2867   "TARGET_NEON"
2868 {
2869   HOST_WIDE_INT magic = INTVAL (operands[3]);
2870   rtx insn;
2871
2872   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2873
2874   if (BYTES_BIG_ENDIAN)
2875     {
2876       /* The intrinsics are defined in terms of a model where the
2877          element ordering in memory is vldm order, whereas the generic
2878          RTL is defined in terms of a model where the element ordering
2879          in memory is array order.  Convert the lane number to conform
2880          to this model.  */
2881       unsigned int elt = INTVAL (operands[2]);
2882       unsigned int reg_nelts
2883         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2884       elt ^= reg_nelts - 1;
2885       operands[2] = GEN_INT (elt);
2886     }
2887
2888   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2889     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2890   else
2891     {
2892       if ((magic & 1) != 0)
2893         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2894                                                        operands[2]);
2895       else
2896         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2897                                                        operands[2]);
2898     }
2899   emit_insn (insn);
2900   DONE;
2901 })
2902
2903 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2904 ; elements.
2905
2906 (define_expand "neon_vget_lanedi"
2907   [(match_operand:DI 0 "s_register_operand" "=r")
2908    (match_operand:DI 1 "s_register_operand" "w")
2909    (match_operand:SI 2 "immediate_operand" "i")
2910    (match_operand:SI 3 "immediate_operand" "i")]
2911   "TARGET_NEON"
2912 {
2913   neon_lane_bounds (operands[2], 0, 1);
2914   emit_move_insn (operands[0], operands[1]);
2915   DONE;
2916 })
2917
2918 (define_expand "neon_vget_lanev2di"
2919   [(match_operand:DI 0 "s_register_operand" "")
2920    (match_operand:V2DI 1 "s_register_operand" "")
2921    (match_operand:SI 2 "immediate_operand" "")
2922    (match_operand:SI 3 "immediate_operand" "")]
2923   "TARGET_NEON"
2924 {
2925   switch (INTVAL (operands[2]))
2926     {
2927     case 0:
2928       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2929       break;
2930     case 1:
2931       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2932       break;
2933     default:
2934       neon_lane_bounds (operands[2], 0, 1);
2935       FAIL;
2936     }
2937   DONE;
2938 })
2939
2940 (define_expand "neon_vset_lane<mode>"
2941   [(match_operand:VDQ 0 "s_register_operand" "=w")
2942    (match_operand:<V_elem> 1 "s_register_operand" "r")
2943    (match_operand:VDQ 2 "s_register_operand" "0")
2944    (match_operand:SI 3 "immediate_operand" "i")]
2945   "TARGET_NEON"
2946 {
2947   unsigned int elt = INTVAL (operands[3]);
2948   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2949
2950   if (BYTES_BIG_ENDIAN)
2951     {
2952       unsigned int reg_nelts
2953         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2954       elt ^= reg_nelts - 1;
2955     }
2956
2957   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2958                                          GEN_INT (1 << elt), operands[2]));
2959   DONE;
2960 })
2961
2962 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2963
2964 (define_expand "neon_vset_lanedi"
2965   [(match_operand:DI 0 "s_register_operand" "=w")
2966    (match_operand:DI 1 "s_register_operand" "r")
2967    (match_operand:DI 2 "s_register_operand" "0")
2968    (match_operand:SI 3 "immediate_operand" "i")]
2969   "TARGET_NEON"
2970 {
2971   neon_lane_bounds (operands[3], 0, 1);
2972   emit_move_insn (operands[0], operands[1]);
2973   DONE;
2974 })
2975
2976 (define_expand "neon_vcreate<mode>"
2977   [(match_operand:VDX 0 "s_register_operand" "")
2978    (match_operand:DI 1 "general_operand" "")]
2979   "TARGET_NEON"
2980 {
2981   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2982   emit_move_insn (operands[0], src);
2983   DONE;
2984 })
2985
2986 (define_insn "neon_vdup_n<mode>"
2987   [(set (match_operand:VX 0 "s_register_operand" "=w")
2988         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2989   "TARGET_NEON"
2990   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2991   ;; Assume this schedules like vmov.
2992   [(set_attr "neon_type" "neon_bp_simple")]
2993 )
2994
2995 (define_insn "neon_vdup_n<mode>"
2996   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2997         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2998   "TARGET_NEON"
2999   "@
3000   vdup.<V_sz_elem>\t%<V_reg>0, %1
3001   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
3002   ;; Assume this schedules like vmov.
3003   [(set_attr "neon_type" "neon_bp_simple")]
3004 )
3005
3006 (define_expand "neon_vdup_ndi"
3007   [(match_operand:DI 0 "s_register_operand" "=w")
3008    (match_operand:DI 1 "s_register_operand" "r")]
3009   "TARGET_NEON"
3010 {
3011   emit_move_insn (operands[0], operands[1]);
3012   DONE;
3013 }
3014 )
3015
3016 (define_insn "neon_vdup_nv2di"
3017   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
3018         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
3019   "TARGET_NEON"
3020   "@
3021   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
3022   vmov\t%e0, %P1\;vmov\t%f0, %P1"
3023   [(set_attr "length" "8")
3024    (set_attr "neon_type" "neon_bp_simple")]
3025 )
3026
3027 (define_insn "neon_vdup_lane<mode>_internal"
3028   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
3029         (vec_duplicate:VDQW
3030           (vec_select:<V_elem>
3031             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3032             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
3033   "TARGET_NEON"
3034 {
3035   if (BYTES_BIG_ENDIAN)
3036     {
3037       int elt = INTVAL (operands[2]);
3038       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
3039       operands[2] = GEN_INT (elt);
3040     }
3041   if (<Is_d_reg>)
3042     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
3043   else
3044     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
3045 }
3046   ;; Assume this schedules like vmov.
3047   [(set_attr "neon_type" "neon_bp_simple")]
3048 )
3049
3050 (define_expand "neon_vdup_lane<mode>"
3051   [(match_operand:VDQW 0 "s_register_operand" "=w")
3052    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3053    (match_operand:SI 2 "immediate_operand" "i")]
3054   "TARGET_NEON"
3055 {
3056   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
3057   if (BYTES_BIG_ENDIAN)
3058     {
3059       unsigned int elt = INTVAL (operands[2]);
3060       unsigned int reg_nelts
3061         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
3062       elt ^= reg_nelts - 1;
3063       operands[2] = GEN_INT (elt);
3064     }
3065     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
3066                                                   operands[2]));
3067     DONE;
3068 })
3069
3070 ; Scalar index is ignored, since only zero is valid here.
3071 (define_expand "neon_vdup_lanedi"
3072   [(match_operand:DI 0 "s_register_operand" "=w")
3073    (match_operand:DI 1 "s_register_operand" "w")
3074    (match_operand:SI 2 "immediate_operand" "i")]
3075   "TARGET_NEON"
3076 {
3077   neon_lane_bounds (operands[2], 0, 1);
3078   emit_move_insn (operands[0], operands[1]);
3079   DONE;
3080 })
3081
3082 ; Likewise for v2di, as the DImode second operand has only a single element.
3083 (define_expand "neon_vdup_lanev2di"
3084   [(match_operand:V2DI 0 "s_register_operand" "=w")
3085    (match_operand:DI 1 "s_register_operand" "w")
3086    (match_operand:SI 2 "immediate_operand" "i")]
3087   "TARGET_NEON"
3088 {
3089   neon_lane_bounds (operands[2], 0, 1);
3090   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
3091   DONE;
3092 })
3093
3094 ; Disabled before reload because we don't want combine doing something silly,
3095 ; but used by the post-reload expansion of neon_vcombine.
3096 (define_insn "*neon_vswp<mode>"
3097   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
3098         (match_operand:VDQX 1 "s_register_operand" "+w"))
3099    (set (match_dup 1) (match_dup 0))]
3100   "TARGET_NEON && reload_completed"
3101   "vswp\t%<V_reg>0, %<V_reg>1"
3102   [(set (attr "neon_type")
3103         (if_then_else (match_test "<Is_d_reg>")
3104                       (const_string "neon_bp_simple")
3105                       (const_string "neon_bp_2cycle")))]
3106 )
3107
3108 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
3109 ;; dest vector.
3110 ;; FIXME: A different implementation of this builtin could make it much
3111 ;; more likely that we wouldn't actually need to output anything (we could make
3112 ;; it so that the reg allocator puts things in the right places magically
3113 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
3114
3115 (define_insn_and_split "neon_vcombine<mode>"
3116   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
3117         (vec_concat:<V_DOUBLE>
3118           (match_operand:VDX 1 "s_register_operand" "w")
3119           (match_operand:VDX 2 "s_register_operand" "w")))]
3120   "TARGET_NEON"
3121   "#"
3122   "&& reload_completed"
3123   [(const_int 0)]
3124 {
3125   neon_split_vcombine (operands);
3126   DONE;
3127 })
3128
3129 (define_expand "neon_vget_high<mode>"
3130   [(match_operand:<V_HALF> 0 "s_register_operand")
3131    (match_operand:VQX 1 "s_register_operand")]
3132   "TARGET_NEON"
3133 {
3134   emit_move_insn (operands[0],
3135                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
3136                                        GET_MODE_SIZE (<V_HALF>mode)));
3137   DONE;
3138 })
3139
3140 (define_expand "neon_vget_low<mode>"
3141   [(match_operand:<V_HALF> 0 "s_register_operand")
3142    (match_operand:VQX 1 "s_register_operand")]
3143   "TARGET_NEON"
3144 {
3145   emit_move_insn (operands[0],
3146                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3147                                        <MODE>mode, 0));
3148   DONE;
3149 })
3150
3151 (define_insn "float<mode><V_cvtto>2"
3152   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3153         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3154   "TARGET_NEON && !flag_rounding_math"
3155   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3156   [(set (attr "neon_type")
3157      (if_then_else (match_test "<Is_d_reg>")
3158                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3159                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3160 )
3161
3162 (define_insn "floatuns<mode><V_cvtto>2"
3163   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3164         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3165   "TARGET_NEON && !flag_rounding_math"
3166   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3167   [(set (attr "neon_type")
3168      (if_then_else (match_test "<Is_d_reg>")
3169                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3170                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3171 )
3172
3173 (define_insn "fix_trunc<mode><V_cvtto>2"
3174   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3175         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3176   "TARGET_NEON"
3177   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3178   [(set (attr "neon_type")
3179      (if_then_else (match_test "<Is_d_reg>")
3180                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3181                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3182 )
3183
3184 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3185   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3186         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3187   "TARGET_NEON"
3188   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3189   [(set (attr "neon_type")
3190      (if_then_else (match_test "<Is_d_reg>")
3191                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3192                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3193 )
3194
3195 (define_insn "neon_vcvt<mode>"
3196   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3197         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3198                            (match_operand:SI 2 "immediate_operand" "i")]
3199                           UNSPEC_VCVT))]
3200   "TARGET_NEON"
3201   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3202   [(set (attr "neon_type")
3203      (if_then_else (match_test "<Is_d_reg>")
3204                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3205                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3206 )
3207
3208 (define_insn "neon_vcvt<mode>"
3209   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3210         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3211                            (match_operand:SI 2 "immediate_operand" "i")]
3212                           UNSPEC_VCVT))]
3213   "TARGET_NEON"
3214   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3215   [(set (attr "neon_type")
3216      (if_then_else (match_test "<Is_d_reg>")
3217                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3218                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3219 )
3220
3221 (define_insn "neon_vcvt_n<mode>"
3222   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3223         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3224                            (match_operand:SI 2 "immediate_operand" "i")
3225                            (match_operand:SI 3 "immediate_operand" "i")]
3226                           UNSPEC_VCVT_N))]
3227   "TARGET_NEON"
3228 {
3229   neon_const_bounds (operands[2], 1, 33);
3230   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3231 }
3232   [(set (attr "neon_type")
3233      (if_then_else (match_test "<Is_d_reg>")
3234                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3235                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3236 )
3237
3238 (define_insn "neon_vcvt_n<mode>"
3239   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3240         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3241                            (match_operand:SI 2 "immediate_operand" "i")
3242                            (match_operand:SI 3 "immediate_operand" "i")]
3243                           UNSPEC_VCVT_N))]
3244   "TARGET_NEON"
3245 {
3246   neon_const_bounds (operands[2], 1, 33);
3247   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3248 }
3249   [(set (attr "neon_type")
3250      (if_then_else (match_test "<Is_d_reg>")
3251                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3252                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3253 )
3254
3255 (define_insn "neon_vmovn<mode>"
3256   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3257         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3258                             (match_operand:SI 2 "immediate_operand" "i")]
3259                            UNSPEC_VMOVN))]
3260   "TARGET_NEON"
3261   "vmovn.<V_if_elem>\t%P0, %q1"
3262   [(set_attr "neon_type" "neon_bp_simple")]
3263 )
3264
3265 (define_insn "neon_vqmovn<mode>"
3266   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3267         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3268                             (match_operand:SI 2 "immediate_operand" "i")]
3269                            UNSPEC_VQMOVN))]
3270   "TARGET_NEON"
3271   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3272   [(set_attr "neon_type" "neon_shift_2")]
3273 )
3274
3275 (define_insn "neon_vqmovun<mode>"
3276   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3277         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3278                             (match_operand:SI 2 "immediate_operand" "i")]
3279                            UNSPEC_VQMOVUN))]
3280   "TARGET_NEON"
3281   "vqmovun.<V_s_elem>\t%P0, %q1"
3282   [(set_attr "neon_type" "neon_shift_2")]
3283 )
3284
3285 (define_insn "neon_vmovl<mode>"
3286   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3287         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3288                            (match_operand:SI 2 "immediate_operand" "i")]
3289                           UNSPEC_VMOVL))]
3290   "TARGET_NEON"
3291   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3292   [(set_attr "neon_type" "neon_shift_1")]
3293 )
3294
3295 (define_insn "neon_vmul_lane<mode>"
3296   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3297         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3298                      (match_operand:VMD 2 "s_register_operand"
3299                                         "<scalar_mul_constraint>")
3300                      (match_operand:SI 3 "immediate_operand" "i")
3301                      (match_operand:SI 4 "immediate_operand" "i")]
3302                     UNSPEC_VMUL_LANE))]
3303   "TARGET_NEON"
3304 {
3305   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3306   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3307 }
3308   [(set (attr "neon_type")
3309      (if_then_else (match_test "<Is_float_mode>")
3310                    (const_string "neon_fp_vmul_ddd")
3311                    (if_then_else (match_test "<Scalar_mul_8_16>")
3312                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3313                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3314 )
3315
3316 (define_insn "neon_vmul_lane<mode>"
3317   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3318         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3319                      (match_operand:<V_HALF> 2 "s_register_operand"
3320                                              "<scalar_mul_constraint>")
3321                      (match_operand:SI 3 "immediate_operand" "i")
3322                      (match_operand:SI 4 "immediate_operand" "i")]
3323                     UNSPEC_VMUL_LANE))]
3324   "TARGET_NEON"
3325 {
3326   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3327   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3328 }
3329   [(set (attr "neon_type")
3330      (if_then_else (match_test "<Is_float_mode>")
3331                    (const_string "neon_fp_vmul_qqd")
3332                    (if_then_else (match_test "<Scalar_mul_8_16>")
3333                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3334                                  (const_string "neon_mul_qqd_32_scalar"))))]
3335 )
3336
3337 (define_insn "neon_vmull_lane<mode>"
3338   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3339         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3340                            (match_operand:VMDI 2 "s_register_operand"
3341                                                "<scalar_mul_constraint>")
3342                            (match_operand:SI 3 "immediate_operand" "i")
3343                            (match_operand:SI 4 "immediate_operand" "i")]
3344                           UNSPEC_VMULL_LANE))]
3345   "TARGET_NEON"
3346 {
3347   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3348   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3349 }
3350   [(set (attr "neon_type")
3351      (if_then_else (match_test "<Scalar_mul_8_16>")
3352                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3353                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3354 )
3355
3356 (define_insn "neon_vqdmull_lane<mode>"
3357   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3358         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3359                            (match_operand:VMDI 2 "s_register_operand"
3360                                                "<scalar_mul_constraint>")
3361                            (match_operand:SI 3 "immediate_operand" "i")
3362                            (match_operand:SI 4 "immediate_operand" "i")]
3363                           UNSPEC_VQDMULL_LANE))]
3364   "TARGET_NEON"
3365 {
3366   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3367   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3368 }
3369   [(set (attr "neon_type")
3370      (if_then_else (match_test "<Scalar_mul_8_16>")
3371                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3372                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3373 )
3374
3375 (define_insn "neon_vqdmulh_lane<mode>"
3376   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3377         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3378                       (match_operand:<V_HALF> 2 "s_register_operand"
3379                                               "<scalar_mul_constraint>")
3380                       (match_operand:SI 3 "immediate_operand" "i")
3381                       (match_operand:SI 4 "immediate_operand" "i")]
3382                       UNSPEC_VQDMULH_LANE))]
3383   "TARGET_NEON"
3384 {
3385   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3386   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3387 }
3388   [(set (attr "neon_type")
3389      (if_then_else (match_test "<Scalar_mul_8_16>")
3390                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3391                    (const_string "neon_mul_qqd_32_scalar")))]
3392 )
3393
3394 (define_insn "neon_vqdmulh_lane<mode>"
3395   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3396         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3397                       (match_operand:VMDI 2 "s_register_operand"
3398                                           "<scalar_mul_constraint>")
3399                       (match_operand:SI 3 "immediate_operand" "i")
3400                       (match_operand:SI 4 "immediate_operand" "i")]
3401                       UNSPEC_VQDMULH_LANE))]
3402   "TARGET_NEON"
3403 {
3404   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3405   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3406 }
3407   [(set (attr "neon_type")
3408      (if_then_else (match_test "<Scalar_mul_8_16>")
3409                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3410                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3411 )
3412
3413 (define_insn "neon_vmla_lane<mode>"
3414   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3415         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3416                      (match_operand:VMD 2 "s_register_operand" "w")
3417                      (match_operand:VMD 3 "s_register_operand"
3418                                         "<scalar_mul_constraint>")
3419                      (match_operand:SI 4 "immediate_operand" "i")
3420                      (match_operand:SI 5 "immediate_operand" "i")]
3421                      UNSPEC_VMLA_LANE))]
3422   "TARGET_NEON"
3423 {
3424   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3425   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3426 }
3427   [(set (attr "neon_type")
3428      (if_then_else (match_test "<Is_float_mode>")
3429                    (const_string "neon_fp_vmla_ddd_scalar")
3430                    (if_then_else (match_test "<Scalar_mul_8_16>")
3431                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3432                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3433 )
3434
3435 (define_insn "neon_vmla_lane<mode>"
3436   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3437         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3438                      (match_operand:VMQ 2 "s_register_operand" "w")
3439                      (match_operand:<V_HALF> 3 "s_register_operand"
3440                                              "<scalar_mul_constraint>")
3441                      (match_operand:SI 4 "immediate_operand" "i")
3442                      (match_operand:SI 5 "immediate_operand" "i")]
3443                      UNSPEC_VMLA_LANE))]
3444   "TARGET_NEON"
3445 {
3446   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3447   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3448 }
3449   [(set (attr "neon_type")
3450      (if_then_else (match_test "<Is_float_mode>")
3451                    (const_string "neon_fp_vmla_qqq_scalar")
3452                    (if_then_else (match_test "<Scalar_mul_8_16>")
3453                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3454                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3455 )
3456
3457 (define_insn "neon_vmlal_lane<mode>"
3458   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3459         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3460                            (match_operand:VMDI 2 "s_register_operand" "w")
3461                            (match_operand:VMDI 3 "s_register_operand"
3462                                                "<scalar_mul_constraint>")
3463                            (match_operand:SI 4 "immediate_operand" "i")
3464                            (match_operand:SI 5 "immediate_operand" "i")]
3465                           UNSPEC_VMLAL_LANE))]
3466   "TARGET_NEON"
3467 {
3468   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3469   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3470 }
3471   [(set (attr "neon_type")
3472      (if_then_else (match_test "<Scalar_mul_8_16>")
3473                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3474                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3475 )
3476
3477 (define_insn "neon_vqdmlal_lane<mode>"
3478   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3479         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3480                            (match_operand:VMDI 2 "s_register_operand" "w")
3481                            (match_operand:VMDI 3 "s_register_operand"
3482                                                "<scalar_mul_constraint>")
3483                            (match_operand:SI 4 "immediate_operand" "i")
3484                            (match_operand:SI 5 "immediate_operand" "i")]
3485                           UNSPEC_VQDMLAL_LANE))]
3486   "TARGET_NEON"
3487 {
3488   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3489   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3490 }
3491   [(set (attr "neon_type")
3492      (if_then_else (match_test "<Scalar_mul_8_16>")
3493                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3494                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3495 )
3496
3497 (define_insn "neon_vmls_lane<mode>"
3498   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3499         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3500                      (match_operand:VMD 2 "s_register_operand" "w")
3501                      (match_operand:VMD 3 "s_register_operand"
3502                                         "<scalar_mul_constraint>")
3503                      (match_operand:SI 4 "immediate_operand" "i")
3504                      (match_operand:SI 5 "immediate_operand" "i")]
3505                     UNSPEC_VMLS_LANE))]
3506   "TARGET_NEON"
3507 {
3508   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3509   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3510 }
3511   [(set (attr "neon_type")
3512      (if_then_else (match_test "<Is_float_mode>")
3513                    (const_string "neon_fp_vmla_ddd_scalar")
3514                    (if_then_else (match_test "<Scalar_mul_8_16>")
3515                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3516                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3517 )
3518
3519 (define_insn "neon_vmls_lane<mode>"
3520   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3521         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3522                      (match_operand:VMQ 2 "s_register_operand" "w")
3523                      (match_operand:<V_HALF> 3 "s_register_operand"
3524                                              "<scalar_mul_constraint>")
3525                      (match_operand:SI 4 "immediate_operand" "i")
3526                      (match_operand:SI 5 "immediate_operand" "i")]
3527                     UNSPEC_VMLS_LANE))]
3528   "TARGET_NEON"
3529 {
3530   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3531   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3532 }
3533   [(set (attr "neon_type")
3534      (if_then_else (match_test "<Is_float_mode>")
3535                    (const_string "neon_fp_vmla_qqq_scalar")
3536                    (if_then_else (match_test "<Scalar_mul_8_16>")
3537                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3538                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3539 )
3540
3541 (define_insn "neon_vmlsl_lane<mode>"
3542   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3543         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3544                            (match_operand:VMDI 2 "s_register_operand" "w")
3545                            (match_operand:VMDI 3 "s_register_operand"
3546                                                "<scalar_mul_constraint>")
3547                            (match_operand:SI 4 "immediate_operand" "i")
3548                            (match_operand:SI 5 "immediate_operand" "i")]
3549                           UNSPEC_VMLSL_LANE))]
3550   "TARGET_NEON"
3551 {
3552   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3553   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3554 }
3555   [(set (attr "neon_type")
3556      (if_then_else (match_test "<Scalar_mul_8_16>")
3557                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3558                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3559 )
3560
3561 (define_insn "neon_vqdmlsl_lane<mode>"
3562   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3563         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3564                            (match_operand:VMDI 2 "s_register_operand" "w")
3565                            (match_operand:VMDI 3 "s_register_operand"
3566                                                "<scalar_mul_constraint>")
3567                            (match_operand:SI 4 "immediate_operand" "i")
3568                            (match_operand:SI 5 "immediate_operand" "i")]
3569                           UNSPEC_VQDMLSL_LANE))]
3570   "TARGET_NEON"
3571 {
3572   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3573   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3574 }
3575   [(set (attr "neon_type")
3576      (if_then_else (match_test "<Scalar_mul_8_16>")
3577                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3578                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3579 )
3580
3581 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3582 ; core register into a temp register, then use a scalar taken from that. This
3583 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3584 ; or extracted from another vector. The latter case it's currently better to
3585 ; use the "_lane" variant, and the former case can probably be implemented
3586 ; using vld1_lane, but that hasn't been done yet.
3587
3588 (define_expand "neon_vmul_n<mode>"
3589   [(match_operand:VMD 0 "s_register_operand" "")
3590    (match_operand:VMD 1 "s_register_operand" "")
3591    (match_operand:<V_elem> 2 "s_register_operand" "")
3592    (match_operand:SI 3 "immediate_operand" "")]
3593   "TARGET_NEON"
3594 {
3595   rtx tmp = gen_reg_rtx (<MODE>mode);
3596   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3597   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3598                                        const0_rtx, const0_rtx));
3599   DONE;
3600 })
3601
3602 (define_expand "neon_vmul_n<mode>"
3603   [(match_operand:VMQ 0 "s_register_operand" "")
3604    (match_operand:VMQ 1 "s_register_operand" "")
3605    (match_operand:<V_elem> 2 "s_register_operand" "")
3606    (match_operand:SI 3 "immediate_operand" "")]
3607   "TARGET_NEON"
3608 {
3609   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3610   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3611   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3612                                        const0_rtx, const0_rtx));
3613   DONE;
3614 })
3615
3616 (define_expand "neon_vmull_n<mode>"
3617   [(match_operand:<V_widen> 0 "s_register_operand" "")
3618    (match_operand:VMDI 1 "s_register_operand" "")
3619    (match_operand:<V_elem> 2 "s_register_operand" "")
3620    (match_operand:SI 3 "immediate_operand" "")]
3621   "TARGET_NEON"
3622 {
3623   rtx tmp = gen_reg_rtx (<MODE>mode);
3624   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3625   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3626                                         const0_rtx, operands[3]));
3627   DONE;
3628 })
3629
3630 (define_expand "neon_vqdmull_n<mode>"
3631   [(match_operand:<V_widen> 0 "s_register_operand" "")
3632    (match_operand:VMDI 1 "s_register_operand" "")
3633    (match_operand:<V_elem> 2 "s_register_operand" "")
3634    (match_operand:SI 3 "immediate_operand" "")]
3635   "TARGET_NEON"
3636 {
3637   rtx tmp = gen_reg_rtx (<MODE>mode);
3638   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3639   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3640                                           const0_rtx, const0_rtx));
3641   DONE;
3642 })
3643
3644 (define_expand "neon_vqdmulh_n<mode>"
3645   [(match_operand:VMDI 0 "s_register_operand" "")
3646    (match_operand:VMDI 1 "s_register_operand" "")
3647    (match_operand:<V_elem> 2 "s_register_operand" "")
3648    (match_operand:SI 3 "immediate_operand" "")]
3649   "TARGET_NEON"
3650 {
3651   rtx tmp = gen_reg_rtx (<MODE>mode);
3652   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3653   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3654                                           const0_rtx, operands[3]));
3655   DONE;
3656 })
3657
3658 (define_expand "neon_vqdmulh_n<mode>"
3659   [(match_operand:VMQI 0 "s_register_operand" "")
3660    (match_operand:VMQI 1 "s_register_operand" "")
3661    (match_operand:<V_elem> 2 "s_register_operand" "")
3662    (match_operand:SI 3 "immediate_operand" "")]
3663   "TARGET_NEON"
3664 {
3665   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3666   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3667   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3668                                           const0_rtx, operands[3]));
3669   DONE;
3670 })
3671
3672 (define_expand "neon_vmla_n<mode>"
3673   [(match_operand:VMD 0 "s_register_operand" "")
3674    (match_operand:VMD 1 "s_register_operand" "")
3675    (match_operand:VMD 2 "s_register_operand" "")
3676    (match_operand:<V_elem> 3 "s_register_operand" "")
3677    (match_operand:SI 4 "immediate_operand" "")]
3678   "TARGET_NEON"
3679 {
3680   rtx tmp = gen_reg_rtx (<MODE>mode);
3681   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3682   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3683                                        tmp, const0_rtx, operands[4]));
3684   DONE;
3685 })
3686
3687 (define_expand "neon_vmla_n<mode>"
3688   [(match_operand:VMQ 0 "s_register_operand" "")
3689    (match_operand:VMQ 1 "s_register_operand" "")
3690    (match_operand:VMQ 2 "s_register_operand" "")
3691    (match_operand:<V_elem> 3 "s_register_operand" "")
3692    (match_operand:SI 4 "immediate_operand" "")]
3693   "TARGET_NEON"
3694 {
3695   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3696   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3697   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3698                                        tmp, const0_rtx, operands[4]));
3699   DONE;
3700 })
3701
3702 (define_expand "neon_vmlal_n<mode>"
3703   [(match_operand:<V_widen> 0 "s_register_operand" "")
3704    (match_operand:<V_widen> 1 "s_register_operand" "")
3705    (match_operand:VMDI 2 "s_register_operand" "")
3706    (match_operand:<V_elem> 3 "s_register_operand" "")
3707    (match_operand:SI 4 "immediate_operand" "")]
3708   "TARGET_NEON"
3709 {
3710   rtx tmp = gen_reg_rtx (<MODE>mode);
3711   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3712   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3713                                         tmp, const0_rtx, operands[4]));
3714   DONE;
3715 })
3716
3717 (define_expand "neon_vqdmlal_n<mode>"
3718   [(match_operand:<V_widen> 0 "s_register_operand" "")
3719    (match_operand:<V_widen> 1 "s_register_operand" "")
3720    (match_operand:VMDI 2 "s_register_operand" "")
3721    (match_operand:<V_elem> 3 "s_register_operand" "")
3722    (match_operand:SI 4 "immediate_operand" "")]
3723   "TARGET_NEON"
3724 {
3725   rtx tmp = gen_reg_rtx (<MODE>mode);
3726   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3727   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3728                                           tmp, const0_rtx, operands[4]));
3729   DONE;
3730 })
3731
3732 (define_expand "neon_vmls_n<mode>"
3733   [(match_operand:VMD 0 "s_register_operand" "")
3734    (match_operand:VMD 1 "s_register_operand" "")
3735    (match_operand:VMD 2 "s_register_operand" "")
3736    (match_operand:<V_elem> 3 "s_register_operand" "")
3737    (match_operand:SI 4 "immediate_operand" "")]
3738   "TARGET_NEON"
3739 {
3740   rtx tmp = gen_reg_rtx (<MODE>mode);
3741   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3742   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3743                                        tmp, const0_rtx, operands[4]));
3744   DONE;
3745 })
3746
3747 (define_expand "neon_vmls_n<mode>"
3748   [(match_operand:VMQ 0 "s_register_operand" "")
3749    (match_operand:VMQ 1 "s_register_operand" "")
3750    (match_operand:VMQ 2 "s_register_operand" "")
3751    (match_operand:<V_elem> 3 "s_register_operand" "")
3752    (match_operand:SI 4 "immediate_operand" "")]
3753   "TARGET_NEON"
3754 {
3755   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3756   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3757   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3758                                        tmp, const0_rtx, operands[4]));
3759   DONE;
3760 })
3761
3762 (define_expand "neon_vmlsl_n<mode>"
3763   [(match_operand:<V_widen> 0 "s_register_operand" "")
3764    (match_operand:<V_widen> 1 "s_register_operand" "")
3765    (match_operand:VMDI 2 "s_register_operand" "")
3766    (match_operand:<V_elem> 3 "s_register_operand" "")
3767    (match_operand:SI 4 "immediate_operand" "")]
3768   "TARGET_NEON"
3769 {
3770   rtx tmp = gen_reg_rtx (<MODE>mode);
3771   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3772   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3773                                         tmp, const0_rtx, operands[4]));
3774   DONE;
3775 })
3776
3777 (define_expand "neon_vqdmlsl_n<mode>"
3778   [(match_operand:<V_widen> 0 "s_register_operand" "")
3779    (match_operand:<V_widen> 1 "s_register_operand" "")
3780    (match_operand:VMDI 2 "s_register_operand" "")
3781    (match_operand:<V_elem> 3 "s_register_operand" "")
3782    (match_operand:SI 4 "immediate_operand" "")]
3783   "TARGET_NEON"
3784 {
3785   rtx tmp = gen_reg_rtx (<MODE>mode);
3786   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3787   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3788                                           tmp, const0_rtx, operands[4]));
3789   DONE;
3790 })
3791
3792 (define_insn "neon_vext<mode>"
3793   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3794         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3795                       (match_operand:VDQX 2 "s_register_operand" "w")
3796                       (match_operand:SI 3 "immediate_operand" "i")]
3797                      UNSPEC_VEXT))]
3798   "TARGET_NEON"
3799 {
3800   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3801   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3802 }
3803   [(set (attr "neon_type")
3804       (if_then_else (match_test "<Is_d_reg>")
3805                     (const_string "neon_bp_simple")
3806                     (const_string "neon_bp_2cycle")))]
3807 )
3808
3809 (define_insn "neon_vrev64<mode>"
3810   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3811         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3812                      (match_operand:SI 2 "immediate_operand" "i")]
3813                     UNSPEC_VREV64))]
3814   "TARGET_NEON"
3815   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3816   [(set_attr "neon_type" "neon_bp_simple")]
3817 )
3818
3819 (define_insn "neon_vrev32<mode>"
3820   [(set (match_operand:VX 0 "s_register_operand" "=w")
3821         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3822                     (match_operand:SI 2 "immediate_operand" "i")]
3823                    UNSPEC_VREV32))]
3824   "TARGET_NEON"
3825   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3826   [(set_attr "neon_type" "neon_bp_simple")]
3827 )
3828
3829 (define_insn "neon_vrev16<mode>"
3830   [(set (match_operand:VE 0 "s_register_operand" "=w")
3831         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3832                     (match_operand:SI 2 "immediate_operand" "i")]
3833                    UNSPEC_VREV16))]
3834   "TARGET_NEON"
3835   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3836   [(set_attr "neon_type" "neon_bp_simple")]
3837 )
3838
3839 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3840 ; allocation. For an intrinsic of form:
3841 ;   rD = vbsl_* (rS, rN, rM)
3842 ; We can use any of:
3843 ;   vbsl rS, rN, rM  (if D = S)
3844 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3845 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3846
3847 (define_insn "neon_vbsl<mode>_internal"
3848   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3849         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3850                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3851                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3852                      UNSPEC_VBSL))]
3853   "TARGET_NEON"
3854   "@
3855   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3856   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3857   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3858   [(set_attr "neon_type" "neon_int_1")]
3859 )
3860
3861 (define_expand "neon_vbsl<mode>"
3862   [(set (match_operand:VDQX 0 "s_register_operand" "")
3863         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3864                       (match_operand:VDQX 2 "s_register_operand" "")
3865                       (match_operand:VDQX 3 "s_register_operand" "")]
3866                      UNSPEC_VBSL))]
3867   "TARGET_NEON"
3868 {
3869   /* We can't alias operands together if they have different modes.  */
3870   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3871 })
3872
3873 (define_insn "neon_vshl<mode>"
3874   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3875         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3876                        (match_operand:VDQIX 2 "s_register_operand" "w")
3877                        (match_operand:SI 3 "immediate_operand" "i")]
3878                       UNSPEC_VSHL))]
3879   "TARGET_NEON"
3880   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3881   [(set (attr "neon_type")
3882       (if_then_else (match_test "<Is_d_reg>")
3883                     (const_string "neon_vshl_ddd")
3884                     (const_string "neon_shift_3")))]
3885 )
3886
3887 (define_insn "neon_vqshl<mode>"
3888   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3889         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3890                        (match_operand:VDQIX 2 "s_register_operand" "w")
3891                        (match_operand:SI 3 "immediate_operand" "i")]
3892                       UNSPEC_VQSHL))]
3893   "TARGET_NEON"
3894   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3895   [(set (attr "neon_type")
3896       (if_then_else (match_test "<Is_d_reg>")
3897                     (const_string "neon_shift_2")
3898                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3899 )
3900
3901 (define_insn "neon_vshr_n<mode>"
3902   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3903         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3904                        (match_operand:SI 2 "immediate_operand" "i")
3905                        (match_operand:SI 3 "immediate_operand" "i")]
3906                       UNSPEC_VSHR_N))]
3907   "TARGET_NEON"
3908 {
3909   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3910   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3911 }
3912   [(set_attr "neon_type" "neon_shift_1")]
3913 )
3914
3915 (define_insn "neon_vshrn_n<mode>"
3916   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3917         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3918                             (match_operand:SI 2 "immediate_operand" "i")
3919                             (match_operand:SI 3 "immediate_operand" "i")]
3920                            UNSPEC_VSHRN_N))]
3921   "TARGET_NEON"
3922 {
3923   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3924   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3925 }
3926   [(set_attr "neon_type" "neon_shift_1")]
3927 )
3928
3929 (define_insn "neon_vqshrn_n<mode>"
3930   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3931         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3932                             (match_operand:SI 2 "immediate_operand" "i")
3933                             (match_operand:SI 3 "immediate_operand" "i")]
3934                            UNSPEC_VQSHRN_N))]
3935   "TARGET_NEON"
3936 {
3937   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3938   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3939 }
3940   [(set_attr "neon_type" "neon_shift_2")]
3941 )
3942
3943 (define_insn "neon_vqshrun_n<mode>"
3944   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3945         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3946                             (match_operand:SI 2 "immediate_operand" "i")
3947                             (match_operand:SI 3 "immediate_operand" "i")]
3948                            UNSPEC_VQSHRUN_N))]
3949   "TARGET_NEON"
3950 {
3951   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3952   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3953 }
3954   [(set_attr "neon_type" "neon_shift_2")]
3955 )
3956
3957 (define_insn "neon_vshl_n<mode>"
3958   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3959         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3960                        (match_operand:SI 2 "immediate_operand" "i")
3961                        (match_operand:SI 3 "immediate_operand" "i")]
3962                       UNSPEC_VSHL_N))]
3963   "TARGET_NEON"
3964 {
3965   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3966   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3967 }
3968   [(set_attr "neon_type" "neon_shift_1")]
3969 )
3970
3971 (define_insn "neon_vqshl_n<mode>"
3972   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3973         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3974                        (match_operand:SI 2 "immediate_operand" "i")
3975                        (match_operand:SI 3 "immediate_operand" "i")]
3976                       UNSPEC_VQSHL_N))]
3977   "TARGET_NEON"
3978 {
3979   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3980   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3981 }
3982   [(set_attr "neon_type" "neon_shift_2")]
3983 )
3984
3985 (define_insn "neon_vqshlu_n<mode>"
3986   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3987         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3988                        (match_operand:SI 2 "immediate_operand" "i")
3989                        (match_operand:SI 3 "immediate_operand" "i")]
3990                       UNSPEC_VQSHLU_N))]
3991   "TARGET_NEON"
3992 {
3993   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3994   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3995 }
3996   [(set_attr "neon_type" "neon_shift_2")]
3997 )
3998
3999 (define_insn "neon_vshll_n<mode>"
4000   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
4001         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
4002                            (match_operand:SI 2 "immediate_operand" "i")
4003                            (match_operand:SI 3 "immediate_operand" "i")]
4004                           UNSPEC_VSHLL_N))]
4005   "TARGET_NEON"
4006 {
4007   /* The boundaries are: 0 < imm <= size.  */
4008   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
4009   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
4010 }
4011   [(set_attr "neon_type" "neon_shift_1")]
4012 )
4013
4014 (define_insn "neon_vsra_n<mode>"
4015   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4016         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4017                        (match_operand:VDQIX 2 "s_register_operand" "w")
4018                        (match_operand:SI 3 "immediate_operand" "i")
4019                        (match_operand:SI 4 "immediate_operand" "i")]
4020                       UNSPEC_VSRA_N))]
4021   "TARGET_NEON"
4022 {
4023   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4024   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4025 }
4026   [(set_attr "neon_type" "neon_vsra_vrsra")]
4027 )
4028
4029 (define_insn "neon_vsri_n<mode>"
4030   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4031         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4032                        (match_operand:VDQIX 2 "s_register_operand" "w")
4033                        (match_operand:SI 3 "immediate_operand" "i")]
4034                       UNSPEC_VSRI))]
4035   "TARGET_NEON"
4036 {
4037   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4038   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4039 }
4040   [(set (attr "neon_type")
4041       (if_then_else (match_test "<Is_d_reg>")
4042                     (const_string "neon_shift_1")
4043                     (const_string "neon_shift_3")))]
4044 )
4045
4046 (define_insn "neon_vsli_n<mode>"
4047   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4048         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4049                        (match_operand:VDQIX 2 "s_register_operand" "w")
4050                        (match_operand:SI 3 "immediate_operand" "i")]
4051                       UNSPEC_VSLI))]
4052   "TARGET_NEON"
4053 {
4054   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
4055   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4056 }
4057   [(set (attr "neon_type")
4058       (if_then_else (match_test "<Is_d_reg>")
4059                     (const_string "neon_shift_1")
4060                     (const_string "neon_shift_3")))]
4061 )
4062
4063 (define_insn "neon_vtbl1v8qi"
4064   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4065         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
4066                       (match_operand:V8QI 2 "s_register_operand" "w")]
4067                      UNSPEC_VTBL))]
4068   "TARGET_NEON"
4069   "vtbl.8\t%P0, {%P1}, %P2"
4070   [(set_attr "neon_type" "neon_bp_2cycle")]
4071 )
4072
4073 (define_insn "neon_vtbl2v8qi"
4074   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4075         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
4076                       (match_operand:V8QI 2 "s_register_operand" "w")]
4077                      UNSPEC_VTBL))]
4078   "TARGET_NEON"
4079 {
4080   rtx ops[4];
4081   int tabbase = REGNO (operands[1]);
4082
4083   ops[0] = operands[0];
4084   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4085   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4086   ops[3] = operands[2];
4087   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
4088
4089   return "";
4090 }
4091   [(set_attr "neon_type" "neon_bp_2cycle")]
4092 )
4093
4094 (define_insn "neon_vtbl3v8qi"
4095   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4096         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
4097                       (match_operand:V8QI 2 "s_register_operand" "w")]
4098                      UNSPEC_VTBL))]
4099   "TARGET_NEON"
4100 {
4101   rtx ops[5];
4102   int tabbase = REGNO (operands[1]);
4103
4104   ops[0] = operands[0];
4105   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4106   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4107   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4108   ops[4] = operands[2];
4109   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4110
4111   return "";
4112 }
4113   [(set_attr "neon_type" "neon_bp_3cycle")]
4114 )
4115
4116 (define_insn "neon_vtbl4v8qi"
4117   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4118         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
4119                       (match_operand:V8QI 2 "s_register_operand" "w")]
4120                      UNSPEC_VTBL))]
4121   "TARGET_NEON"
4122 {
4123   rtx ops[6];
4124   int tabbase = REGNO (operands[1]);
4125
4126   ops[0] = operands[0];
4127   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4128   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4129   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4130   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4131   ops[5] = operands[2];
4132   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4133
4134   return "";
4135 }
4136   [(set_attr "neon_type" "neon_bp_3cycle")]
4137 )
4138
4139 ;; These three are used by the vec_perm infrastructure for V16QImode.
4140 (define_insn_and_split "neon_vtbl1v16qi"
4141   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4142         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
4143                        (match_operand:V16QI 2 "s_register_operand" "w")]
4144                       UNSPEC_VTBL))]
4145   "TARGET_NEON"
4146   "#"
4147   "&& reload_completed"
4148   [(const_int 0)]
4149 {
4150   rtx op0, op1, op2, part0, part2;
4151   unsigned ofs;
4152
4153   op0 = operands[0];
4154   op1 = gen_lowpart (TImode, operands[1]);
4155   op2 = operands[2];
4156
4157   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4158   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4159   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4160   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4161
4162   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4163   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4164   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4165   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4166   DONE;
4167 })
4168
4169 (define_insn_and_split "neon_vtbl2v16qi"
4170   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4171         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4172                        (match_operand:V16QI 2 "s_register_operand" "w")]
4173                       UNSPEC_VTBL))]
4174   "TARGET_NEON"
4175   "#"
4176   "&& reload_completed"
4177   [(const_int 0)]
4178 {
4179   rtx op0, op1, op2, part0, part2;
4180   unsigned ofs;
4181
4182   op0 = operands[0];
4183   op1 = operands[1];
4184   op2 = operands[2];
4185
4186   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4187   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4188   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4189   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4190
4191   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4192   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4193   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4194   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4195   DONE;
4196 })
4197
4198 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4199 ;; handle quad-word input modes, producing octa-word output modes.  But
4200 ;; that requires us to add support for octa-word vector modes in moves.
4201 ;; That seems overkill for this one use in vec_perm.
4202 (define_insn_and_split "neon_vcombinev16qi"
4203   [(set (match_operand:OI 0 "s_register_operand" "=w")
4204         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4205                     (match_operand:V16QI 2 "s_register_operand" "w")]
4206                    UNSPEC_VCONCAT))]
4207   "TARGET_NEON"
4208   "#"
4209   "&& reload_completed"
4210   [(const_int 0)]
4211 {
4212   neon_split_vcombine (operands);
4213   DONE;
4214 })
4215
4216 (define_insn "neon_vtbx1v8qi"
4217   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4218         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4219                       (match_operand:V8QI 2 "s_register_operand" "w")
4220                       (match_operand:V8QI 3 "s_register_operand" "w")]
4221                      UNSPEC_VTBX))]
4222   "TARGET_NEON"
4223   "vtbx.8\t%P0, {%P2}, %P3"
4224   [(set_attr "neon_type" "neon_bp_2cycle")]
4225 )
4226
4227 (define_insn "neon_vtbx2v8qi"
4228   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4229         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4230                       (match_operand:TI 2 "s_register_operand" "w")
4231                       (match_operand:V8QI 3 "s_register_operand" "w")]
4232                      UNSPEC_VTBX))]
4233   "TARGET_NEON"
4234 {
4235   rtx ops[4];
4236   int tabbase = REGNO (operands[2]);
4237
4238   ops[0] = operands[0];
4239   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4240   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4241   ops[3] = operands[3];
4242   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4243
4244   return "";
4245 }
4246   [(set_attr "neon_type" "neon_bp_2cycle")]
4247 )
4248
4249 (define_insn "neon_vtbx3v8qi"
4250   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4251         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4252                       (match_operand:EI 2 "s_register_operand" "w")
4253                       (match_operand:V8QI 3 "s_register_operand" "w")]
4254                      UNSPEC_VTBX))]
4255   "TARGET_NEON"
4256 {
4257   rtx ops[5];
4258   int tabbase = REGNO (operands[2]);
4259
4260   ops[0] = operands[0];
4261   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4262   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4263   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4264   ops[4] = operands[3];
4265   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4266
4267   return "";
4268 }
4269   [(set_attr "neon_type" "neon_bp_3cycle")]
4270 )
4271
4272 (define_insn "neon_vtbx4v8qi"
4273   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4274         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4275                       (match_operand:OI 2 "s_register_operand" "w")
4276                       (match_operand:V8QI 3 "s_register_operand" "w")]
4277                      UNSPEC_VTBX))]
4278   "TARGET_NEON"
4279 {
4280   rtx ops[6];
4281   int tabbase = REGNO (operands[2]);
4282
4283   ops[0] = operands[0];
4284   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4285   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4286   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4287   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4288   ops[5] = operands[3];
4289   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4290
4291   return "";
4292 }
4293   [(set_attr "neon_type" "neon_bp_3cycle")]
4294 )
4295
4296 (define_expand "neon_vtrn<mode>_internal"
4297   [(parallel
4298     [(set (match_operand:VDQW 0 "s_register_operand" "")
4299           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4300                         (match_operand:VDQW 2 "s_register_operand" "")]
4301            UNSPEC_VTRN1))
4302      (set (match_operand:VDQW 3 "s_register_operand" "")
4303           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VTRN2))])]
4304   "TARGET_NEON"
4305   ""
4306 )
4307
4308 ;; Note: Different operand numbering to handle tied registers correctly.
4309 (define_insn "*neon_vtrn<mode>_insn"
4310   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4311         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4312                       (match_operand:VDQW 3 "s_register_operand" "2")]
4313                      UNSPEC_VTRN1))
4314    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4315          (unspec:VDQW [(match_dup 1) (match_dup 3)]
4316                      UNSPEC_VTRN2))]
4317   "TARGET_NEON"
4318   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4319   [(set (attr "neon_type")
4320       (if_then_else (match_test "<Is_d_reg>")
4321                     (const_string "neon_bp_simple")
4322                     (const_string "neon_bp_3cycle")))]
4323 )
4324
4325 (define_expand "neon_vtrn<mode>"
4326   [(match_operand:SI 0 "s_register_operand" "r")
4327    (match_operand:VDQW 1 "s_register_operand" "w")
4328    (match_operand:VDQW 2 "s_register_operand" "w")]
4329   "TARGET_NEON"
4330 {
4331   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4332                               operands[0], operands[1], operands[2]);
4333   DONE;
4334 })
4335
4336 (define_expand "neon_vzip<mode>_internal"
4337   [(parallel
4338     [(set (match_operand:VDQW 0 "s_register_operand" "")
4339           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4340                         (match_operand:VDQW 2 "s_register_operand" "")]
4341                        UNSPEC_VZIP1))
4342     (set (match_operand:VDQW 3 "s_register_operand" "")
4343          (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VZIP2))])]
4344   "TARGET_NEON"
4345   ""
4346 )
4347
4348 ;; Note: Different operand numbering to handle tied registers correctly.
4349 (define_insn "*neon_vzip<mode>_insn"
4350   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4351         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4352                       (match_operand:VDQW 3 "s_register_operand" "2")]
4353                      UNSPEC_VZIP1))
4354    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4355         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4356                      UNSPEC_VZIP2))]
4357   "TARGET_NEON"
4358   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4359   [(set (attr "neon_type")
4360       (if_then_else (match_test "<Is_d_reg>")
4361                     (const_string "neon_bp_simple")
4362                     (const_string "neon_bp_3cycle")))]
4363 )
4364
4365 (define_expand "neon_vzip<mode>"
4366   [(match_operand:SI 0 "s_register_operand" "r")
4367    (match_operand:VDQW 1 "s_register_operand" "w")
4368    (match_operand:VDQW 2 "s_register_operand" "w")]
4369   "TARGET_NEON"
4370 {
4371   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4372                               operands[0], operands[1], operands[2]);
4373   DONE;
4374 })
4375
4376 (define_expand "neon_vuzp<mode>_internal"
4377   [(parallel
4378     [(set (match_operand:VDQW 0 "s_register_operand" "")
4379           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4380                         (match_operand:VDQW 2 "s_register_operand" "")]
4381            UNSPEC_VUZP1))
4382      (set (match_operand:VDQW 3 "s_register_operand" "")
4383           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VUZP2))])]
4384   "TARGET_NEON"
4385   ""
4386 )
4387
4388 ;; Note: Different operand numbering to handle tied registers correctly.
4389 (define_insn "*neon_vuzp<mode>_insn"
4390   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4391         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4392                       (match_operand:VDQW 3 "s_register_operand" "2")]
4393                      UNSPEC_VUZP1))
4394    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4395         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4396                      UNSPEC_VUZP2))]
4397   "TARGET_NEON"
4398   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4399   [(set (attr "neon_type")
4400       (if_then_else (match_test "<Is_d_reg>")
4401                     (const_string "neon_bp_simple")
4402                     (const_string "neon_bp_3cycle")))]
4403 )
4404
4405 (define_expand "neon_vuzp<mode>"
4406   [(match_operand:SI 0 "s_register_operand" "r")
4407    (match_operand:VDQW 1 "s_register_operand" "w")
4408    (match_operand:VDQW 2 "s_register_operand" "w")]
4409   "TARGET_NEON"
4410 {
4411   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4412                               operands[0], operands[1], operands[2]);
4413   DONE;
4414 })
4415
4416 (define_expand "neon_vreinterpretv8qi<mode>"
4417   [(match_operand:V8QI 0 "s_register_operand" "")
4418    (match_operand:VDX 1 "s_register_operand" "")]
4419   "TARGET_NEON"
4420 {
4421   neon_reinterpret (operands[0], operands[1]);
4422   DONE;
4423 })
4424
4425 (define_expand "neon_vreinterpretv4hi<mode>"
4426   [(match_operand:V4HI 0 "s_register_operand" "")
4427    (match_operand:VDX 1 "s_register_operand" "")]
4428   "TARGET_NEON"
4429 {
4430   neon_reinterpret (operands[0], operands[1]);
4431   DONE;
4432 })
4433
4434 (define_expand "neon_vreinterpretv2si<mode>"
4435   [(match_operand:V2SI 0 "s_register_operand" "")
4436    (match_operand:VDX 1 "s_register_operand" "")]
4437   "TARGET_NEON"
4438 {
4439   neon_reinterpret (operands[0], operands[1]);
4440   DONE;
4441 })
4442
4443 (define_expand "neon_vreinterpretv2sf<mode>"
4444   [(match_operand:V2SF 0 "s_register_operand" "")
4445    (match_operand:VDX 1 "s_register_operand" "")]
4446   "TARGET_NEON"
4447 {
4448   neon_reinterpret (operands[0], operands[1]);
4449   DONE;
4450 })
4451
4452 (define_expand "neon_vreinterpretdi<mode>"
4453   [(match_operand:DI 0 "s_register_operand" "")
4454    (match_operand:VDX 1 "s_register_operand" "")]
4455   "TARGET_NEON"
4456 {
4457   neon_reinterpret (operands[0], operands[1]);
4458   DONE;
4459 })
4460
4461 (define_expand "neon_vreinterpretv16qi<mode>"
4462   [(match_operand:V16QI 0 "s_register_operand" "")
4463    (match_operand:VQX 1 "s_register_operand" "")]
4464   "TARGET_NEON"
4465 {
4466   neon_reinterpret (operands[0], operands[1]);
4467   DONE;
4468 })
4469
4470 (define_expand "neon_vreinterpretv8hi<mode>"
4471   [(match_operand:V8HI 0 "s_register_operand" "")
4472    (match_operand:VQX 1 "s_register_operand" "")]
4473   "TARGET_NEON"
4474 {
4475   neon_reinterpret (operands[0], operands[1]);
4476   DONE;
4477 })
4478
4479 (define_expand "neon_vreinterpretv4si<mode>"
4480   [(match_operand:V4SI 0 "s_register_operand" "")
4481    (match_operand:VQX 1 "s_register_operand" "")]
4482   "TARGET_NEON"
4483 {
4484   neon_reinterpret (operands[0], operands[1]);
4485   DONE;
4486 })
4487
4488 (define_expand "neon_vreinterpretv4sf<mode>"
4489   [(match_operand:V4SF 0 "s_register_operand" "")
4490    (match_operand:VQX 1 "s_register_operand" "")]
4491   "TARGET_NEON"
4492 {
4493   neon_reinterpret (operands[0], operands[1]);
4494   DONE;
4495 })
4496
4497 (define_expand "neon_vreinterpretv2di<mode>"
4498   [(match_operand:V2DI 0 "s_register_operand" "")
4499    (match_operand:VQX 1 "s_register_operand" "")]
4500   "TARGET_NEON"
4501 {
4502   neon_reinterpret (operands[0], operands[1]);
4503   DONE;
4504 })
4505
4506 (define_expand "vec_load_lanes<mode><mode>"
4507   [(set (match_operand:VDQX 0 "s_register_operand")
4508         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4509                      UNSPEC_VLD1))]
4510   "TARGET_NEON")
4511
4512 (define_insn "neon_vld1<mode>"
4513   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4514         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4515                     UNSPEC_VLD1))]
4516   "TARGET_NEON"
4517   "vld1.<V_sz_elem>\t%h0, %A1"
4518   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4519 )
4520
4521 (define_insn "neon_vld1_lane<mode>"
4522   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4523         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4524                      (match_operand:VDX 2 "s_register_operand" "0")
4525                      (match_operand:SI 3 "immediate_operand" "i")]
4526                     UNSPEC_VLD1_LANE))]
4527   "TARGET_NEON"
4528 {
4529   HOST_WIDE_INT lane = INTVAL (operands[3]);
4530   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4531   if (lane < 0 || lane >= max)
4532     error ("lane out of range");
4533   if (max == 1)
4534     return "vld1.<V_sz_elem>\t%P0, %A1";
4535   else
4536     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4537 }
4538   [(set (attr "neon_type")
4539       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4540                     (const_string "neon_vld1_1_2_regs")
4541                     (const_string "neon_vld1_vld2_lane")))]
4542 )
4543
4544 (define_insn "neon_vld1_lane<mode>"
4545   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4546         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4547                      (match_operand:VQX 2 "s_register_operand" "0")
4548                      (match_operand:SI 3 "immediate_operand" "i")]
4549                     UNSPEC_VLD1_LANE))]
4550   "TARGET_NEON"
4551 {
4552   HOST_WIDE_INT lane = INTVAL (operands[3]);
4553   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4554   int regno = REGNO (operands[0]);
4555   if (lane < 0 || lane >= max)
4556     error ("lane out of range");
4557   else if (lane >= max / 2)
4558     {
4559       lane -= max / 2;
4560       regno += 2;
4561       operands[3] = GEN_INT (lane);
4562     }
4563   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4564   if (max == 2)
4565     return "vld1.<V_sz_elem>\t%P0, %A1";
4566   else
4567     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4568 }
4569   [(set (attr "neon_type")
4570       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4571                     (const_string "neon_vld1_1_2_regs")
4572                     (const_string "neon_vld1_vld2_lane")))]
4573 )
4574
4575 (define_insn "neon_vld1_dup<mode>"
4576   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4577         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4578   "TARGET_NEON"
4579 {
4580   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4581     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4582   else
4583     return "vld1.<V_sz_elem>\t%h0, %A1";
4584 }
4585   [(set (attr "neon_type")
4586       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4587                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4588                     (const_string "neon_vld1_1_2_regs")))]
4589 )
4590
4591 (define_insn "neon_vld1_dup<mode>"
4592   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4593         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4594   "TARGET_NEON"
4595 {
4596   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4597 }
4598   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4599 )
4600
4601 (define_insn_and_split "neon_vld1_dupv2di"
4602    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4603     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4604    "TARGET_NEON"
4605    "#"
4606    "&& reload_completed"
4607    [(const_int 0)]
4608    {
4609     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4610     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4611     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4612     DONE;
4613     }
4614   [(set_attr "length" "8")
4615    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4616 )
4617
4618 (define_expand "vec_store_lanes<mode><mode>"
4619   [(set (match_operand:VDQX 0 "neon_struct_operand")
4620         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4621                      UNSPEC_VST1))]
4622   "TARGET_NEON")
4623
4624 (define_insn "neon_vst1<mode>"
4625   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4626         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4627                      UNSPEC_VST1))]
4628   "TARGET_NEON"
4629   "vst1.<V_sz_elem>\t%h1, %A0"
4630   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4631
4632 (define_insn "neon_vst1_lane<mode>"
4633   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4634         (unspec:<V_elem>
4635           [(match_operand:VDX 1 "s_register_operand" "w")
4636            (match_operand:SI 2 "immediate_operand" "i")]
4637           UNSPEC_VST1_LANE))]
4638   "TARGET_NEON"
4639 {
4640   HOST_WIDE_INT lane = INTVAL (operands[2]);
4641   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4642   if (lane < 0 || lane >= max)
4643     error ("lane out of range");
4644   if (max == 1)
4645     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4646   else
4647     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4648 }
4649   [(set (attr "neon_type")
4650       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4651                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4652                     (const_string "neon_vst1_vst2_lane")))])
4653
4654 (define_insn "neon_vst1_lane<mode>"
4655   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4656         (unspec:<V_elem>
4657           [(match_operand:VQX 1 "s_register_operand" "w")
4658            (match_operand:SI 2 "immediate_operand" "i")]
4659           UNSPEC_VST1_LANE))]
4660   "TARGET_NEON"
4661 {
4662   HOST_WIDE_INT lane = INTVAL (operands[2]);
4663   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4664   int regno = REGNO (operands[1]);
4665   if (lane < 0 || lane >= max)
4666     error ("lane out of range");
4667   else if (lane >= max / 2)
4668     {
4669       lane -= max / 2;
4670       regno += 2;
4671       operands[2] = GEN_INT (lane);
4672     }
4673   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4674   if (max == 2)
4675     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4676   else
4677     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4678 }
4679   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4680 )
4681
4682 (define_expand "vec_load_lanesti<mode>"
4683   [(set (match_operand:TI 0 "s_register_operand")
4684         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4685                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4686                    UNSPEC_VLD2))]
4687   "TARGET_NEON")
4688
4689 (define_insn "neon_vld2<mode>"
4690   [(set (match_operand:TI 0 "s_register_operand" "=w")
4691         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4692                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4693                    UNSPEC_VLD2))]
4694   "TARGET_NEON"
4695 {
4696   if (<V_sz_elem> == 64)
4697     return "vld1.64\t%h0, %A1";
4698   else
4699     return "vld2.<V_sz_elem>\t%h0, %A1";
4700 }
4701   [(set (attr "neon_type")
4702       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4703                     (const_string "neon_vld1_1_2_regs")
4704                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4705 )
4706
4707 (define_expand "vec_load_lanesoi<mode>"
4708   [(set (match_operand:OI 0 "s_register_operand")
4709         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4710                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4711                    UNSPEC_VLD2))]
4712   "TARGET_NEON")
4713
4714 (define_insn "neon_vld2<mode>"
4715   [(set (match_operand:OI 0 "s_register_operand" "=w")
4716         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4717                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4718                    UNSPEC_VLD2))]
4719   "TARGET_NEON"
4720   "vld2.<V_sz_elem>\t%h0, %A1"
4721   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4722
4723 (define_insn "neon_vld2_lane<mode>"
4724   [(set (match_operand:TI 0 "s_register_operand" "=w")
4725         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4726                     (match_operand:TI 2 "s_register_operand" "0")
4727                     (match_operand:SI 3 "immediate_operand" "i")
4728                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4729                    UNSPEC_VLD2_LANE))]
4730   "TARGET_NEON"
4731 {
4732   HOST_WIDE_INT lane = INTVAL (operands[3]);
4733   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4734   int regno = REGNO (operands[0]);
4735   rtx ops[4];
4736   if (lane < 0 || lane >= max)
4737     error ("lane out of range");
4738   ops[0] = gen_rtx_REG (DImode, regno);
4739   ops[1] = gen_rtx_REG (DImode, regno + 2);
4740   ops[2] = operands[1];
4741   ops[3] = operands[3];
4742   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4743   return "";
4744 }
4745   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4746 )
4747
4748 (define_insn "neon_vld2_lane<mode>"
4749   [(set (match_operand:OI 0 "s_register_operand" "=w")
4750         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4751                     (match_operand:OI 2 "s_register_operand" "0")
4752                     (match_operand:SI 3 "immediate_operand" "i")
4753                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4754                    UNSPEC_VLD2_LANE))]
4755   "TARGET_NEON"
4756 {
4757   HOST_WIDE_INT lane = INTVAL (operands[3]);
4758   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4759   int regno = REGNO (operands[0]);
4760   rtx ops[4];
4761   if (lane < 0 || lane >= max)
4762     error ("lane out of range");
4763   else if (lane >= max / 2)
4764     {
4765       lane -= max / 2;
4766       regno += 2;
4767     }
4768   ops[0] = gen_rtx_REG (DImode, regno);
4769   ops[1] = gen_rtx_REG (DImode, regno + 4);
4770   ops[2] = operands[1];
4771   ops[3] = GEN_INT (lane);
4772   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4773   return "";
4774 }
4775   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4776 )
4777
4778 (define_insn "neon_vld2_dup<mode>"
4779   [(set (match_operand:TI 0 "s_register_operand" "=w")
4780         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4781                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4782                    UNSPEC_VLD2_DUP))]
4783   "TARGET_NEON"
4784 {
4785   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4786     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4787   else
4788     return "vld1.<V_sz_elem>\t%h0, %A1";
4789 }
4790   [(set (attr "neon_type")
4791       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4792                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4793                     (const_string "neon_vld1_1_2_regs")))]
4794 )
4795
4796 (define_expand "vec_store_lanesti<mode>"
4797   [(set (match_operand:TI 0 "neon_struct_operand")
4798         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4799                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4800                    UNSPEC_VST2))]
4801   "TARGET_NEON")
4802
4803 (define_insn "neon_vst2<mode>"
4804   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4805         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4806                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4807                    UNSPEC_VST2))]
4808   "TARGET_NEON"
4809 {
4810   if (<V_sz_elem> == 64)
4811     return "vst1.64\t%h1, %A0";
4812   else
4813     return "vst2.<V_sz_elem>\t%h1, %A0";
4814 }
4815   [(set (attr "neon_type")
4816       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4817                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4818                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4819 )
4820
4821 (define_expand "vec_store_lanesoi<mode>"
4822   [(set (match_operand:OI 0 "neon_struct_operand")
4823         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4824                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4825                    UNSPEC_VST2))]
4826   "TARGET_NEON")
4827
4828 (define_insn "neon_vst2<mode>"
4829   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4830         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4831                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4832                    UNSPEC_VST2))]
4833   "TARGET_NEON"
4834   "vst2.<V_sz_elem>\t%h1, %A0"
4835   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4836 )
4837
4838 (define_insn "neon_vst2_lane<mode>"
4839   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4840         (unspec:<V_two_elem>
4841           [(match_operand:TI 1 "s_register_operand" "w")
4842            (match_operand:SI 2 "immediate_operand" "i")
4843            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4844           UNSPEC_VST2_LANE))]
4845   "TARGET_NEON"
4846 {
4847   HOST_WIDE_INT lane = INTVAL (operands[2]);
4848   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4849   int regno = REGNO (operands[1]);
4850   rtx ops[4];
4851   if (lane < 0 || lane >= max)
4852     error ("lane out of range");
4853   ops[0] = operands[0];
4854   ops[1] = gen_rtx_REG (DImode, regno);
4855   ops[2] = gen_rtx_REG (DImode, regno + 2);
4856   ops[3] = operands[2];
4857   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4858   return "";
4859 }
4860   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4861 )
4862
4863 (define_insn "neon_vst2_lane<mode>"
4864   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4865         (unspec:<V_two_elem>
4866            [(match_operand:OI 1 "s_register_operand" "w")
4867             (match_operand:SI 2 "immediate_operand" "i")
4868             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4869            UNSPEC_VST2_LANE))]
4870   "TARGET_NEON"
4871 {
4872   HOST_WIDE_INT lane = INTVAL (operands[2]);
4873   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4874   int regno = REGNO (operands[1]);
4875   rtx ops[4];
4876   if (lane < 0 || lane >= max)
4877     error ("lane out of range");
4878   else if (lane >= max / 2)
4879     {
4880       lane -= max / 2;
4881       regno += 2;
4882     }
4883   ops[0] = operands[0];
4884   ops[1] = gen_rtx_REG (DImode, regno);
4885   ops[2] = gen_rtx_REG (DImode, regno + 4);
4886   ops[3] = GEN_INT (lane);
4887   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4888   return "";
4889 }
4890   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4891 )
4892
4893 (define_expand "vec_load_lanesei<mode>"
4894   [(set (match_operand:EI 0 "s_register_operand")
4895         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4896                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4897                    UNSPEC_VLD3))]
4898   "TARGET_NEON")
4899
4900 (define_insn "neon_vld3<mode>"
4901   [(set (match_operand:EI 0 "s_register_operand" "=w")
4902         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4903                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4904                    UNSPEC_VLD3))]
4905   "TARGET_NEON"
4906 {
4907   if (<V_sz_elem> == 64)
4908     return "vld1.64\t%h0, %A1";
4909   else
4910     return "vld3.<V_sz_elem>\t%h0, %A1";
4911 }
4912   [(set (attr "neon_type")
4913       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4914                     (const_string "neon_vld1_1_2_regs")
4915                     (const_string "neon_vld3_vld4")))]
4916 )
4917
4918 (define_expand "vec_load_lanesci<mode>"
4919   [(match_operand:CI 0 "s_register_operand")
4920    (match_operand:CI 1 "neon_struct_operand")
4921    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4922   "TARGET_NEON"
4923 {
4924   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4925   DONE;
4926 })
4927
4928 (define_expand "neon_vld3<mode>"
4929   [(match_operand:CI 0 "s_register_operand")
4930    (match_operand:CI 1 "neon_struct_operand")
4931    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4932   "TARGET_NEON"
4933 {
4934   rtx mem;
4935
4936   mem = adjust_address (operands[1], EImode, 0);
4937   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4938   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4939   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4940   DONE;
4941 })
4942
4943 (define_insn "neon_vld3qa<mode>"
4944   [(set (match_operand:CI 0 "s_register_operand" "=w")
4945         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4946                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4947                    UNSPEC_VLD3A))]
4948   "TARGET_NEON"
4949 {
4950   int regno = REGNO (operands[0]);
4951   rtx ops[4];
4952   ops[0] = gen_rtx_REG (DImode, regno);
4953   ops[1] = gen_rtx_REG (DImode, regno + 4);
4954   ops[2] = gen_rtx_REG (DImode, regno + 8);
4955   ops[3] = operands[1];
4956   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4957   return "";
4958 }
4959   [(set_attr "neon_type" "neon_vld3_vld4")]
4960 )
4961
4962 (define_insn "neon_vld3qb<mode>"
4963   [(set (match_operand:CI 0 "s_register_operand" "=w")
4964         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4965                     (match_operand:CI 2 "s_register_operand" "0")
4966                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4967                    UNSPEC_VLD3B))]
4968   "TARGET_NEON"
4969 {
4970   int regno = REGNO (operands[0]);
4971   rtx ops[4];
4972   ops[0] = gen_rtx_REG (DImode, regno + 2);
4973   ops[1] = gen_rtx_REG (DImode, regno + 6);
4974   ops[2] = gen_rtx_REG (DImode, regno + 10);
4975   ops[3] = operands[1];
4976   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4977   return "";
4978 }
4979   [(set_attr "neon_type" "neon_vld3_vld4")]
4980 )
4981
4982 (define_insn "neon_vld3_lane<mode>"
4983   [(set (match_operand:EI 0 "s_register_operand" "=w")
4984         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4985                     (match_operand:EI 2 "s_register_operand" "0")
4986                     (match_operand:SI 3 "immediate_operand" "i")
4987                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4988                    UNSPEC_VLD3_LANE))]
4989   "TARGET_NEON"
4990 {
4991   HOST_WIDE_INT lane = INTVAL (operands[3]);
4992   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4993   int regno = REGNO (operands[0]);
4994   rtx ops[5];
4995   if (lane < 0 || lane >= max)
4996     error ("lane out of range");
4997   ops[0] = gen_rtx_REG (DImode, regno);
4998   ops[1] = gen_rtx_REG (DImode, regno + 2);
4999   ops[2] = gen_rtx_REG (DImode, regno + 4);
5000   ops[3] = operands[1];
5001   ops[4] = operands[3];
5002   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5003                    ops);
5004   return "";
5005 }
5006   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5007 )
5008
5009 (define_insn "neon_vld3_lane<mode>"
5010   [(set (match_operand:CI 0 "s_register_operand" "=w")
5011         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5012                     (match_operand:CI 2 "s_register_operand" "0")
5013                     (match_operand:SI 3 "immediate_operand" "i")
5014                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5015                    UNSPEC_VLD3_LANE))]
5016   "TARGET_NEON"
5017 {
5018   HOST_WIDE_INT lane = INTVAL (operands[3]);
5019   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5020   int regno = REGNO (operands[0]);
5021   rtx ops[5];
5022   if (lane < 0 || lane >= max)
5023     error ("lane out of range");
5024   else if (lane >= max / 2)
5025     {
5026       lane -= max / 2;
5027       regno += 2;
5028     }
5029   ops[0] = gen_rtx_REG (DImode, regno);
5030   ops[1] = gen_rtx_REG (DImode, regno + 4);
5031   ops[2] = gen_rtx_REG (DImode, regno + 8);
5032   ops[3] = operands[1];
5033   ops[4] = GEN_INT (lane);
5034   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5035                    ops);
5036   return "";
5037 }
5038   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5039 )
5040
5041 (define_insn "neon_vld3_dup<mode>"
5042   [(set (match_operand:EI 0 "s_register_operand" "=w")
5043         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5044                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5045                    UNSPEC_VLD3_DUP))]
5046   "TARGET_NEON"
5047 {
5048   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5049     {
5050       int regno = REGNO (operands[0]);
5051       rtx ops[4];
5052       ops[0] = gen_rtx_REG (DImode, regno);
5053       ops[1] = gen_rtx_REG (DImode, regno + 2);
5054       ops[2] = gen_rtx_REG (DImode, regno + 4);
5055       ops[3] = operands[1];
5056       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
5057       return "";
5058     }
5059   else
5060     return "vld1.<V_sz_elem>\t%h0, %A1";
5061 }
5062   [(set (attr "neon_type")
5063       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5064                     (const_string "neon_vld3_vld4_all_lanes")
5065                     (const_string "neon_vld1_1_2_regs")))])
5066
5067 (define_expand "vec_store_lanesei<mode>"
5068   [(set (match_operand:EI 0 "neon_struct_operand")
5069         (unspec:EI [(match_operand:EI 1 "s_register_operand")
5070                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5071                    UNSPEC_VST3))]
5072   "TARGET_NEON")
5073
5074 (define_insn "neon_vst3<mode>"
5075   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5076         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
5077                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5078                    UNSPEC_VST3))]
5079   "TARGET_NEON"
5080 {
5081   if (<V_sz_elem> == 64)
5082     return "vst1.64\t%h1, %A0";
5083   else
5084     return "vst3.<V_sz_elem>\t%h1, %A0";
5085 }
5086   [(set (attr "neon_type")
5087       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5088                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5089                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
5090
5091 (define_expand "vec_store_lanesci<mode>"
5092   [(match_operand:CI 0 "neon_struct_operand")
5093    (match_operand:CI 1 "s_register_operand")
5094    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5095   "TARGET_NEON"
5096 {
5097   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
5098   DONE;
5099 })
5100
5101 (define_expand "neon_vst3<mode>"
5102   [(match_operand:CI 0 "neon_struct_operand")
5103    (match_operand:CI 1 "s_register_operand")
5104    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5105   "TARGET_NEON"
5106 {
5107   rtx mem;
5108
5109   mem = adjust_address (operands[0], EImode, 0);
5110   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
5111   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
5112   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
5113   DONE;
5114 })
5115
5116 (define_insn "neon_vst3qa<mode>"
5117   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5118         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5119                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5120                    UNSPEC_VST3A))]
5121   "TARGET_NEON"
5122 {
5123   int regno = REGNO (operands[1]);
5124   rtx ops[4];
5125   ops[0] = operands[0];
5126   ops[1] = gen_rtx_REG (DImode, regno);
5127   ops[2] = gen_rtx_REG (DImode, regno + 4);
5128   ops[3] = gen_rtx_REG (DImode, regno + 8);
5129   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5130   return "";
5131 }
5132   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5133 )
5134
5135 (define_insn "neon_vst3qb<mode>"
5136   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5137         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5138                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5139                    UNSPEC_VST3B))]
5140   "TARGET_NEON"
5141 {
5142   int regno = REGNO (operands[1]);
5143   rtx ops[4];
5144   ops[0] = operands[0];
5145   ops[1] = gen_rtx_REG (DImode, regno + 2);
5146   ops[2] = gen_rtx_REG (DImode, regno + 6);
5147   ops[3] = gen_rtx_REG (DImode, regno + 10);
5148   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5149   return "";
5150 }
5151   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5152 )
5153
5154 (define_insn "neon_vst3_lane<mode>"
5155   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5156         (unspec:<V_three_elem>
5157            [(match_operand:EI 1 "s_register_operand" "w")
5158             (match_operand:SI 2 "immediate_operand" "i")
5159             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5160            UNSPEC_VST3_LANE))]
5161   "TARGET_NEON"
5162 {
5163   HOST_WIDE_INT lane = INTVAL (operands[2]);
5164   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5165   int regno = REGNO (operands[1]);
5166   rtx ops[5];
5167   if (lane < 0 || lane >= max)
5168     error ("lane out of range");
5169   ops[0] = operands[0];
5170   ops[1] = gen_rtx_REG (DImode, regno);
5171   ops[2] = gen_rtx_REG (DImode, regno + 2);
5172   ops[3] = gen_rtx_REG (DImode, regno + 4);
5173   ops[4] = operands[2];
5174   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5175                    ops);
5176   return "";
5177 }
5178   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5179 )
5180
5181 (define_insn "neon_vst3_lane<mode>"
5182   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5183         (unspec:<V_three_elem>
5184            [(match_operand:CI 1 "s_register_operand" "w")
5185             (match_operand:SI 2 "immediate_operand" "i")
5186             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5187            UNSPEC_VST3_LANE))]
5188   "TARGET_NEON"
5189 {
5190   HOST_WIDE_INT lane = INTVAL (operands[2]);
5191   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5192   int regno = REGNO (operands[1]);
5193   rtx ops[5];
5194   if (lane < 0 || lane >= max)
5195     error ("lane out of range");
5196   else if (lane >= max / 2)
5197     {
5198       lane -= max / 2;
5199       regno += 2;
5200     }
5201   ops[0] = operands[0];
5202   ops[1] = gen_rtx_REG (DImode, regno);
5203   ops[2] = gen_rtx_REG (DImode, regno + 4);
5204   ops[3] = gen_rtx_REG (DImode, regno + 8);
5205   ops[4] = GEN_INT (lane);
5206   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5207                    ops);
5208   return "";
5209 }
5210 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5211
5212 (define_expand "vec_load_lanesoi<mode>"
5213   [(set (match_operand:OI 0 "s_register_operand")
5214         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5215                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5216                    UNSPEC_VLD4))]
5217   "TARGET_NEON")
5218
5219 (define_insn "neon_vld4<mode>"
5220   [(set (match_operand:OI 0 "s_register_operand" "=w")
5221         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5222                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5223                    UNSPEC_VLD4))]
5224   "TARGET_NEON"
5225 {
5226   if (<V_sz_elem> == 64)
5227     return "vld1.64\t%h0, %A1";
5228   else
5229     return "vld4.<V_sz_elem>\t%h0, %A1";
5230 }
5231   [(set (attr "neon_type")
5232       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5233                     (const_string "neon_vld1_1_2_regs")
5234                     (const_string "neon_vld3_vld4")))]
5235 )
5236
5237 (define_expand "vec_load_lanesxi<mode>"
5238   [(match_operand:XI 0 "s_register_operand")
5239    (match_operand:XI 1 "neon_struct_operand")
5240    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5241   "TARGET_NEON"
5242 {
5243   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5244   DONE;
5245 })
5246
5247 (define_expand "neon_vld4<mode>"
5248   [(match_operand:XI 0 "s_register_operand")
5249    (match_operand:XI 1 "neon_struct_operand")
5250    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5251   "TARGET_NEON"
5252 {
5253   rtx mem;
5254
5255   mem = adjust_address (operands[1], OImode, 0);
5256   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5257   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5258   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5259   DONE;
5260 })
5261
5262 (define_insn "neon_vld4qa<mode>"
5263   [(set (match_operand:XI 0 "s_register_operand" "=w")
5264         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5265                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5266                    UNSPEC_VLD4A))]
5267   "TARGET_NEON"
5268 {
5269   int regno = REGNO (operands[0]);
5270   rtx ops[5];
5271   ops[0] = gen_rtx_REG (DImode, regno);
5272   ops[1] = gen_rtx_REG (DImode, regno + 4);
5273   ops[2] = gen_rtx_REG (DImode, regno + 8);
5274   ops[3] = gen_rtx_REG (DImode, regno + 12);
5275   ops[4] = operands[1];
5276   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5277   return "";
5278 }
5279   [(set_attr "neon_type" "neon_vld3_vld4")]
5280 )
5281
5282 (define_insn "neon_vld4qb<mode>"
5283   [(set (match_operand:XI 0 "s_register_operand" "=w")
5284         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5285                     (match_operand:XI 2 "s_register_operand" "0")
5286                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5287                    UNSPEC_VLD4B))]
5288   "TARGET_NEON"
5289 {
5290   int regno = REGNO (operands[0]);
5291   rtx ops[5];
5292   ops[0] = gen_rtx_REG (DImode, regno + 2);
5293   ops[1] = gen_rtx_REG (DImode, regno + 6);
5294   ops[2] = gen_rtx_REG (DImode, regno + 10);
5295   ops[3] = gen_rtx_REG (DImode, regno + 14);
5296   ops[4] = operands[1];
5297   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5298   return "";
5299 }
5300   [(set_attr "neon_type" "neon_vld3_vld4")]
5301 )
5302
5303 (define_insn "neon_vld4_lane<mode>"
5304   [(set (match_operand:OI 0 "s_register_operand" "=w")
5305         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5306                     (match_operand:OI 2 "s_register_operand" "0")
5307                     (match_operand:SI 3 "immediate_operand" "i")
5308                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5309                    UNSPEC_VLD4_LANE))]
5310   "TARGET_NEON"
5311 {
5312   HOST_WIDE_INT lane = INTVAL (operands[3]);
5313   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5314   int regno = REGNO (operands[0]);
5315   rtx ops[6];
5316   if (lane < 0 || lane >= max)
5317     error ("lane out of range");
5318   ops[0] = gen_rtx_REG (DImode, regno);
5319   ops[1] = gen_rtx_REG (DImode, regno + 2);
5320   ops[2] = gen_rtx_REG (DImode, regno + 4);
5321   ops[3] = gen_rtx_REG (DImode, regno + 6);
5322   ops[4] = operands[1];
5323   ops[5] = operands[3];
5324   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5325                    ops);
5326   return "";
5327 }
5328   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5329 )
5330
5331 (define_insn "neon_vld4_lane<mode>"
5332   [(set (match_operand:XI 0 "s_register_operand" "=w")
5333         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5334                     (match_operand:XI 2 "s_register_operand" "0")
5335                     (match_operand:SI 3 "immediate_operand" "i")
5336                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5337                    UNSPEC_VLD4_LANE))]
5338   "TARGET_NEON"
5339 {
5340   HOST_WIDE_INT lane = INTVAL (operands[3]);
5341   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5342   int regno = REGNO (operands[0]);
5343   rtx ops[6];
5344   if (lane < 0 || lane >= max)
5345     error ("lane out of range");
5346   else if (lane >= max / 2)
5347     {
5348       lane -= max / 2;
5349       regno += 2;
5350     }
5351   ops[0] = gen_rtx_REG (DImode, regno);
5352   ops[1] = gen_rtx_REG (DImode, regno + 4);
5353   ops[2] = gen_rtx_REG (DImode, regno + 8);
5354   ops[3] = gen_rtx_REG (DImode, regno + 12);
5355   ops[4] = operands[1];
5356   ops[5] = GEN_INT (lane);
5357   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5358                    ops);
5359   return "";
5360 }
5361   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5362 )
5363
5364 (define_insn "neon_vld4_dup<mode>"
5365   [(set (match_operand:OI 0 "s_register_operand" "=w")
5366         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5367                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5368                    UNSPEC_VLD4_DUP))]
5369   "TARGET_NEON"
5370 {
5371   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5372     {
5373       int regno = REGNO (operands[0]);
5374       rtx ops[5];
5375       ops[0] = gen_rtx_REG (DImode, regno);
5376       ops[1] = gen_rtx_REG (DImode, regno + 2);
5377       ops[2] = gen_rtx_REG (DImode, regno + 4);
5378       ops[3] = gen_rtx_REG (DImode, regno + 6);
5379       ops[4] = operands[1];
5380       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5381                        ops);
5382       return "";
5383     }
5384   else
5385     return "vld1.<V_sz_elem>\t%h0, %A1";
5386 }
5387   [(set (attr "neon_type")
5388       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5389                     (const_string "neon_vld3_vld4_all_lanes")
5390                     (const_string "neon_vld1_1_2_regs")))]
5391 )
5392
5393 (define_expand "vec_store_lanesoi<mode>"
5394   [(set (match_operand:OI 0 "neon_struct_operand")
5395         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5396                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5397                    UNSPEC_VST4))]
5398   "TARGET_NEON")
5399
5400 (define_insn "neon_vst4<mode>"
5401   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5402         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5403                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5404                    UNSPEC_VST4))]
5405   "TARGET_NEON"
5406 {
5407   if (<V_sz_elem> == 64)
5408     return "vst1.64\t%h1, %A0";
5409   else
5410     return "vst4.<V_sz_elem>\t%h1, %A0";
5411 }
5412   [(set (attr "neon_type")
5413       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5414                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5415                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5416 )
5417
5418 (define_expand "vec_store_lanesxi<mode>"
5419   [(match_operand:XI 0 "neon_struct_operand")
5420    (match_operand:XI 1 "s_register_operand")
5421    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5422   "TARGET_NEON"
5423 {
5424   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5425   DONE;
5426 })
5427
5428 (define_expand "neon_vst4<mode>"
5429   [(match_operand:XI 0 "neon_struct_operand")
5430    (match_operand:XI 1 "s_register_operand")
5431    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5432   "TARGET_NEON"
5433 {
5434   rtx mem;
5435
5436   mem = adjust_address (operands[0], OImode, 0);
5437   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5438   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5439   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5440   DONE;
5441 })
5442
5443 (define_insn "neon_vst4qa<mode>"
5444   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5445         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5446                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5447                    UNSPEC_VST4A))]
5448   "TARGET_NEON"
5449 {
5450   int regno = REGNO (operands[1]);
5451   rtx ops[5];
5452   ops[0] = operands[0];
5453   ops[1] = gen_rtx_REG (DImode, regno);
5454   ops[2] = gen_rtx_REG (DImode, regno + 4);
5455   ops[3] = gen_rtx_REG (DImode, regno + 8);
5456   ops[4] = gen_rtx_REG (DImode, regno + 12);
5457   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5458   return "";
5459 }
5460   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5461 )
5462
5463 (define_insn "neon_vst4qb<mode>"
5464   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5465         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5466                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5467                    UNSPEC_VST4B))]
5468   "TARGET_NEON"
5469 {
5470   int regno = REGNO (operands[1]);
5471   rtx ops[5];
5472   ops[0] = operands[0];
5473   ops[1] = gen_rtx_REG (DImode, regno + 2);
5474   ops[2] = gen_rtx_REG (DImode, regno + 6);
5475   ops[3] = gen_rtx_REG (DImode, regno + 10);
5476   ops[4] = gen_rtx_REG (DImode, regno + 14);
5477   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5478   return "";
5479 }
5480   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5481 )
5482
5483 (define_insn "neon_vst4_lane<mode>"
5484   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5485         (unspec:<V_four_elem>
5486            [(match_operand:OI 1 "s_register_operand" "w")
5487             (match_operand:SI 2 "immediate_operand" "i")
5488             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5489            UNSPEC_VST4_LANE))]
5490   "TARGET_NEON"
5491 {
5492   HOST_WIDE_INT lane = INTVAL (operands[2]);
5493   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5494   int regno = REGNO (operands[1]);
5495   rtx ops[6];
5496   if (lane < 0 || lane >= max)
5497     error ("lane out of range");
5498   ops[0] = operands[0];
5499   ops[1] = gen_rtx_REG (DImode, regno);
5500   ops[2] = gen_rtx_REG (DImode, regno + 2);
5501   ops[3] = gen_rtx_REG (DImode, regno + 4);
5502   ops[4] = gen_rtx_REG (DImode, regno + 6);
5503   ops[5] = operands[2];
5504   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5505                    ops);
5506   return "";
5507 }
5508   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5509 )
5510
5511 (define_insn "neon_vst4_lane<mode>"
5512   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5513         (unspec:<V_four_elem>
5514            [(match_operand:XI 1 "s_register_operand" "w")
5515             (match_operand:SI 2 "immediate_operand" "i")
5516             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5517            UNSPEC_VST4_LANE))]
5518   "TARGET_NEON"
5519 {
5520   HOST_WIDE_INT lane = INTVAL (operands[2]);
5521   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5522   int regno = REGNO (operands[1]);
5523   rtx ops[6];
5524   if (lane < 0 || lane >= max)
5525     error ("lane out of range");
5526   else if (lane >= max / 2)
5527     {
5528       lane -= max / 2;
5529       regno += 2;
5530     }
5531   ops[0] = operands[0];
5532   ops[1] = gen_rtx_REG (DImode, regno);
5533   ops[2] = gen_rtx_REG (DImode, regno + 4);
5534   ops[3] = gen_rtx_REG (DImode, regno + 8);
5535   ops[4] = gen_rtx_REG (DImode, regno + 12);
5536   ops[5] = GEN_INT (lane);
5537   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5538                    ops);
5539   return "";
5540 }
5541   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5542 )
5543
5544 (define_expand "neon_vand<mode>"
5545   [(match_operand:VDQX 0 "s_register_operand" "")
5546    (match_operand:VDQX 1 "s_register_operand" "")
5547    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5548    (match_operand:SI 3 "immediate_operand" "")]
5549   "TARGET_NEON"
5550 {
5551   emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5552   DONE;
5553 })
5554
5555 (define_expand "neon_vorr<mode>"
5556   [(match_operand:VDQX 0 "s_register_operand" "")
5557    (match_operand:VDQX 1 "s_register_operand" "")
5558    (match_operand:VDQX 2 "neon_logic_op2" "")
5559    (match_operand:SI 3 "immediate_operand" "")]
5560   "TARGET_NEON"
5561 {
5562   emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5563   DONE;
5564 })
5565
5566 (define_expand "neon_veor<mode>"
5567   [(match_operand:VDQX 0 "s_register_operand" "")
5568    (match_operand:VDQX 1 "s_register_operand" "")
5569    (match_operand:VDQX 2 "s_register_operand" "")
5570    (match_operand:SI 3 "immediate_operand" "")]
5571   "TARGET_NEON"
5572 {
5573   emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5574   DONE;
5575 })
5576
5577 (define_expand "neon_vbic<mode>"
5578   [(match_operand:VDQX 0 "s_register_operand" "")
5579    (match_operand:VDQX 1 "s_register_operand" "")
5580    (match_operand:VDQX 2 "neon_logic_op2" "")
5581    (match_operand:SI 3 "immediate_operand" "")]
5582   "TARGET_NEON"
5583 {
5584   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5585   DONE;
5586 })
5587
5588 (define_expand "neon_vorn<mode>"
5589   [(match_operand:VDQX 0 "s_register_operand" "")
5590    (match_operand:VDQX 1 "s_register_operand" "")
5591    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5592    (match_operand:SI 3 "immediate_operand" "")]
5593   "TARGET_NEON"
5594 {
5595   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5596   DONE;
5597 })
5598
5599 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5600   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5601         (SE:<V_unpack> (vec_select:<V_HALF>
5602                           (match_operand:VU 1 "register_operand" "w")
5603                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5604   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5605   "vmovl.<US><V_sz_elem> %q0, %e1"
5606   [(set_attr "neon_type" "neon_shift_1")]
5607 )
5608
5609 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5610   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5611         (SE:<V_unpack> (vec_select:<V_HALF>
5612                           (match_operand:VU 1 "register_operand" "w")
5613                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5614   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5615   "vmovl.<US><V_sz_elem> %q0, %f1"
5616   [(set_attr "neon_type" "neon_shift_1")]
5617 )
5618
5619 (define_expand "vec_unpack<US>_hi_<mode>"
5620   [(match_operand:<V_unpack> 0 "register_operand" "")
5621    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5622  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5623   {
5624    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5625    rtx t1;
5626    int i;
5627    for (i = 0; i < (<V_mode_nunits>/2); i++)
5628      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5629
5630    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5631    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5632                                                  operands[1],
5633                                                  t1));
5634    DONE;
5635   }
5636 )
5637
5638 (define_expand "vec_unpack<US>_lo_<mode>"
5639   [(match_operand:<V_unpack> 0 "register_operand" "")
5640    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5641  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5642   {
5643    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5644    rtx t1;
5645    int i;
5646    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5647      RTVEC_ELT (v, i) = GEN_INT (i);
5648    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5649    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5650                                                  operands[1],
5651                                                  t1));
5652    DONE;
5653   }
5654 )
5655
5656 (define_insn "neon_vec_<US>mult_lo_<mode>"
5657  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5658        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5659                            (match_operand:VU 1 "register_operand" "w")
5660                            (match_operand:VU 2 "vect_par_constant_low" "")))
5661                         (SE:<V_unpack> (vec_select:<V_HALF>
5662                            (match_operand:VU 3 "register_operand" "w")
5663                            (match_dup 2)))))]
5664   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5665   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5666   [(set_attr "neon_type" "neon_shift_1")]
5667 )
5668
5669 (define_expand "vec_widen_<US>mult_lo_<mode>"
5670   [(match_operand:<V_unpack> 0 "register_operand" "")
5671    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5672    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5673  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5674  {
5675    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5676    rtx t1;
5677    int i;
5678    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5679      RTVEC_ELT (v, i) = GEN_INT (i);
5680    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5681
5682    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5683                                                operands[1],
5684                                                t1,
5685                                                operands[2]));
5686    DONE;
5687  }
5688 )
5689
5690 (define_insn "neon_vec_<US>mult_hi_<mode>"
5691  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5692       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5693                             (match_operand:VU 1 "register_operand" "w")
5694                             (match_operand:VU 2 "vect_par_constant_high" "")))
5695                        (SE:<V_unpack> (vec_select:<V_HALF>
5696                             (match_operand:VU 3 "register_operand" "w")
5697                             (match_dup 2)))))]
5698   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5699   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5700   [(set_attr "neon_type" "neon_shift_1")]
5701 )
5702
5703 (define_expand "vec_widen_<US>mult_hi_<mode>"
5704   [(match_operand:<V_unpack> 0 "register_operand" "")
5705    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5706    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5707  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5708  {
5709    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5710    rtx t1;
5711    int i;
5712    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5713      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5714    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5715
5716    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5717                                                operands[1],
5718                                                t1,
5719                                                operands[2]));
5720    DONE;
5721
5722  }
5723 )
5724
5725 (define_insn "neon_vec_<US>shiftl_<mode>"
5726  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5727        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5728        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5729   "TARGET_NEON"
5730 {
5731   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5732 }
5733   [(set_attr "neon_type" "neon_shift_1")]
5734 )
5735
5736 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5737   [(match_operand:<V_unpack> 0 "register_operand" "")
5738    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5739    (match_operand:SI 2 "immediate_operand" "i")]
5740  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5741  {
5742   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5743                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5744                 operands[2]));
5745    DONE;
5746  }
5747 )
5748
5749 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5750   [(match_operand:<V_unpack> 0 "register_operand" "")
5751    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5752    (match_operand:SI 2 "immediate_operand" "i")]
5753  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5754  {
5755   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5756                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5757                                      GET_MODE_SIZE (<V_HALF>mode)),
5758                 operands[2]));
5759    DONE;
5760  }
5761 )
5762
5763 ;; Vectorize for non-neon-quad case
5764 (define_insn "neon_unpack<US>_<mode>"
5765  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5766        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5767  "TARGET_NEON"
5768  "vmovl.<US><V_sz_elem> %q0, %P1"
5769   [(set_attr "neon_type" "neon_shift_1")]
5770 )
5771
5772 (define_expand "vec_unpack<US>_lo_<mode>"
5773  [(match_operand:<V_double_width> 0 "register_operand" "")
5774   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5775  "TARGET_NEON"
5776 {
5777   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5778   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5779   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5780
5781   DONE;
5782 }
5783 )
5784
5785 (define_expand "vec_unpack<US>_hi_<mode>"
5786  [(match_operand:<V_double_width> 0 "register_operand" "")
5787   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5788  "TARGET_NEON"
5789 {
5790   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5791   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5792   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5793
5794   DONE;
5795 }
5796 )
5797
5798 (define_insn "neon_vec_<US>mult_<mode>"
5799  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5800        (mult:<V_widen> (SE:<V_widen>
5801                            (match_operand:VDI 1 "register_operand" "w"))
5802                        (SE:<V_widen>
5803                            (match_operand:VDI 2 "register_operand" "w"))))]
5804   "TARGET_NEON"
5805   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5806   [(set_attr "neon_type" "neon_shift_1")]
5807 )
5808
5809 (define_expand "vec_widen_<US>mult_hi_<mode>"
5810   [(match_operand:<V_double_width> 0 "register_operand" "")
5811    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5812    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5813  "TARGET_NEON"
5814  {
5815    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5816    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5817    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5818
5819    DONE;
5820
5821  }
5822 )
5823
5824 (define_expand "vec_widen_<US>mult_lo_<mode>"
5825   [(match_operand:<V_double_width> 0 "register_operand" "")
5826    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5827    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5828  "TARGET_NEON"
5829  {
5830    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5831    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5832    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5833
5834    DONE;
5835
5836  }
5837 )
5838
5839 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5840  [(match_operand:<V_double_width> 0 "register_operand" "")
5841    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5842    (match_operand:SI 2 "immediate_operand" "i")]
5843  "TARGET_NEON"
5844  {
5845    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5846    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5847    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5848
5849    DONE;
5850  }
5851 )
5852
5853 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5854   [(match_operand:<V_double_width> 0 "register_operand" "")
5855    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5856    (match_operand:SI 2 "immediate_operand" "i")]
5857  "TARGET_NEON"
5858  {
5859    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5860    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5861    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5862
5863    DONE;
5864  }
5865 )
5866
5867 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5868 ; because the ordering of vector elements in Q registers is different from what
5869 ; the semantics of the instructions require.
5870
5871 (define_insn "vec_pack_trunc_<mode>"
5872  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5873        (vec_concat:<V_narrow_pack>
5874                 (truncate:<V_narrow>
5875                         (match_operand:VN 1 "register_operand" "w"))
5876                 (truncate:<V_narrow>
5877                         (match_operand:VN 2 "register_operand" "w"))))]
5878  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5879  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5880  [(set_attr "neon_type" "neon_shift_1")
5881   (set_attr "length" "8")]
5882 )
5883
5884 ;; For the non-quad case.
5885 (define_insn "neon_vec_pack_trunc_<mode>"
5886  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5887        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5888  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5889  "vmovn.i<V_sz_elem>\t%P0, %q1"
5890  [(set_attr "neon_type" "neon_shift_1")]
5891 )
5892
5893 (define_expand "vec_pack_trunc_<mode>"
5894  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5895   (match_operand:VSHFT 1 "register_operand" "")
5896   (match_operand:VSHFT 2 "register_operand")]
5897  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5898 {
5899   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5900
5901   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5902   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5903   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5904   DONE;
5905 })
5906
5907 (define_insn "neon_vabd<mode>_2"
5908  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5909        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5910                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5911  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5912  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5913  [(set (attr "neon_type")
5914        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5915                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5916                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5917                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5918                      (const_string "neon_int_5")))]
5919 )
5920
5921 (define_insn "neon_vabd<mode>_3"
5922  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5923        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5924                              (match_operand:VDQ 2 "s_register_operand" "w")]
5925                  UNSPEC_VSUB)))]
5926  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5927  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5928  [(set (attr "neon_type")
5929        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5930                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5931                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5932                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5933                      (const_string "neon_int_5")))]
5934 )
5935
5936 ;; Copy from core-to-neon regs, then extend, not vice-versa
5937
5938 (define_split
5939   [(set (match_operand:DI 0 "s_register_operand" "")
5940         (sign_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5941   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5942   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5943    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 32)))]
5944   {
5945     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5946   })
5947
5948 (define_split
5949   [(set (match_operand:DI 0 "s_register_operand" "")
5950         (sign_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5951   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5952   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5953    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 48)))]
5954   {
5955     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5956   })
5957
5958 (define_split
5959   [(set (match_operand:DI 0 "s_register_operand" "")
5960         (sign_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5961   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5962   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5963    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))]
5964   {
5965     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5966   })
5967
5968 (define_split
5969   [(set (match_operand:DI 0 "s_register_operand" "")
5970         (zero_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5971   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5972   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5973    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 32)))]
5974   {
5975     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5976   })
5977
5978 (define_split
5979   [(set (match_operand:DI 0 "s_register_operand" "")
5980         (zero_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5981   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5982   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5983    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 48)))]
5984   {
5985     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5986   })
5987
5988 (define_split
5989   [(set (match_operand:DI 0 "s_register_operand" "")
5990         (zero_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5991   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5992   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5993    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
5994   {
5995     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5996   })