gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Uxi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       rtx ops[2];
  45       int regno = (which_alternative == 7)
  46                   ? REGNO (operands[1]) : REGNO (operands[0]);
  47
  48       ops[0] = operands[0];
  49       ops[1] = operands[1];
  50       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode)
  51         {
  52           if (which_alternative == 7)
  53             {
  54               ops[1] = gen_rtx_REG (DImode, regno);
  55               output_asm_insn ("vstr.64\t%P1, %E0",ops);
  56             }
  57           else
  58             {
  59               ops[0] = gen_rtx_REG (DImode, regno);
  60               output_asm_insn ("vldr.64\t%P0, %E1",ops);
  61             }
  62         }
  63       else if (<MODE>mode == TImode)
  64         {
  65           if (which_alternative == 7)
  66             output_asm_insn ("vstr.64\t%q1, %E0",ops);
  67           else
  68             output_asm_insn ("vldr.64\t%q0, %E1",ops);
  69         }
  70       else
  71         {
  72           if (which_alternative == 7)
  73             {
  74               ops[1] = gen_rtx_REG (TImode, regno);
  75               output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0",ops);
  76             }
  77           else
  78             {
  79               ops[0] = gen_rtx_REG (TImode, regno);
  80               output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1",ops);
  81             }
  82         }
  83       return "";
  84     }
  85   switch (which_alternative)
  86     {
  87     case 0:
  88       return "vmov\t%q0, %q1";
  89     case 1:
  90       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  91     case 2:
  92       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  93     case 5:
  94       return output_move_quad (operands);
  95     case 8:
  96         return output_move_neon (operands);
  97     default:
  98       gcc_unreachable ();
  99       return "";
 100     }
 101 }
 102   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
 103    (set_attr "length" "4,8,8,4,8,8,4,4,4")
 104    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
 105    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
 106
 107 (define_insn "*mve_mov<mode>"
 108   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 109         (vec_duplicate:MVE_types
 110           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 111   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 112 {
 113   if (which_alternative == 0)
 114     return "vdup.<V_sz_elem>\t%q0, %1";
 115   return "vmov.<V_sz_elem>\t%q0, %1";
 116 }
 117   [(set_attr "length" "4,4")
 118    (set_attr "type" "mve_move,mve_move")])
 119
 120 ;;
 121 ;; [vst4q])
 122 ;;
 123 (define_insn "mve_vst4q<mode>"
 124   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 125         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 126                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 127          VST4Q))
 128   ]
 129   "TARGET_HAVE_MVE"
 130 {
 131    rtx ops[6];
 132    int regno = REGNO (operands[1]);
 133    ops[0] = gen_rtx_REG (TImode, regno);
 134    ops[1] = gen_rtx_REG (TImode, regno+4);
 135    ops[2] = gen_rtx_REG (TImode, regno+8);
 136    ops[3] = gen_rtx_REG (TImode, regno+12);
 137    rtx reg  = operands[0];
 138    while (reg && !REG_P (reg))
 139     reg = XEXP (reg, 0);
 140    gcc_assert (REG_P (reg));
 141    ops[4] = reg;
 142    ops[5] = operands[0];
 143    /* Here in first three instructions data is stored to ops[4]'s location but
 144       in the fourth instruction data is stored to operands[0], this is to
 145       support the writeback.  */
 146    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 147                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 148                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 149                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 150    return "";
 151 }
 152   [(set_attr "length" "16")])
 153
 154 ;;
 155 ;; [vrndq_m_f])
 156 ;;
 157 (define_insn "mve_vrndq_m_f<mode>"
 158   [
 159    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 160         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 161                        (match_operand:MVE_0 2 "s_register_operand" "w")
 162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 163          VRNDQ_M_F))
 164   ]
 165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 166   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 167   [(set_attr "type" "mve_move")
 168    (set_attr "length""8")])
 169
 170 ;;
 171 ;; [vrndxq_f])
 172 ;;
 173 (define_insn "mve_vrndxq_f<mode>"
 174   [
 175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 177          VRNDXQ_F))
 178   ]
 179   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 180   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 181   [(set_attr "type" "mve_move")
 182 ])
 183
 184 ;;
 185 ;; [vrndq_f])
 186 ;;
 187 (define_insn "mve_vrndq_f<mode>"
 188   [
 189    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 190         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 191          VRNDQ_F))
 192   ]
 193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 194   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 195   [(set_attr "type" "mve_move")
 196 ])
 197
 198 ;;
 199 ;; [vrndpq_f])
 200 ;;
 201 (define_insn "mve_vrndpq_f<mode>"
 202   [
 203    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 204         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 205          VRNDPQ_F))
 206   ]
 207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 208   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 209   [(set_attr "type" "mve_move")
 210 ])
 211
 212 ;;
 213 ;; [vrndnq_f])
 214 ;;
 215 (define_insn "mve_vrndnq_f<mode>"
 216   [
 217    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 218         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 219          VRNDNQ_F))
 220   ]
 221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 222   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 223   [(set_attr "type" "mve_move")
 224 ])
 225
 226 ;;
 227 ;; [vrndmq_f])
 228 ;;
 229 (define_insn "mve_vrndmq_f<mode>"
 230   [
 231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 232         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 233          VRNDMQ_F))
 234   ]
 235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 236   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 237   [(set_attr "type" "mve_move")
 238 ])
 239
 240 ;;
 241 ;; [vrndaq_f])
 242 ;;
 243 (define_insn "mve_vrndaq_f<mode>"
 244   [
 245    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 246         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 247          VRNDAQ_F))
 248   ]
 249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 250   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 251   [(set_attr "type" "mve_move")
 252 ])
 253
 254 ;;
 255 ;; [vrev64q_f])
 256 ;;
 257 (define_insn "mve_vrev64q_f<mode>"
 258   [
 259    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 260         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 261          VREV64Q_F))
 262   ]
 263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 264   "vrev64.%#<V_sz_elem> %q0, %q1"
 265   [(set_attr "type" "mve_move")
 266 ])
 267
 268 ;;
 269 ;; [vnegq_f])
 270 ;;
 271 (define_insn "mve_vnegq_f<mode>"
 272   [
 273    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 274         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 275   ]
 276   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 277   "vneg.f%#<V_sz_elem>  %q0, %q1"
 278   [(set_attr "type" "mve_move")
 279 ])
 280
 281 ;;
 282 ;; [vdupq_n_f])
 283 ;;
 284 (define_insn "mve_vdupq_n_f<mode>"
 285   [
 286    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 287         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 288          VDUPQ_N_F))
 289   ]
 290   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 291   "vdup.%#<V_sz_elem>   %q0, %1"
 292   [(set_attr "type" "mve_move")
 293 ])
 294
 295 ;;
 296 ;; [vabsq_f])
 297 ;;
 298 (define_insn "mve_vabsq_f<mode>"
 299   [
 300    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 301         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 302          VABSQ_F))
 303   ]
 304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 305   "vabs.f%#<V_sz_elem>  %q0, %q1"
 306   [(set_attr "type" "mve_move")
 307 ])
 308
 309 ;;
 310 ;; [vrev32q_f])
 311 ;;
 312 (define_insn "mve_vrev32q_fv8hf"
 313   [
 314    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 315         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 316          VREV32Q_F))
 317   ]
 318   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 319   "vrev32.16 %q0, %q1"
 320   [(set_attr "type" "mve_move")
 321 ])
 322 ;;
 323 ;; [vcvttq_f32_f16])
 324 ;;
 325 (define_insn "mve_vcvttq_f32_f16v4sf"
 326   [
 327    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 328         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 329          VCVTTQ_F32_F16))
 330   ]
 331   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 332   "vcvtt.f32.f16 %q0, %q1"
 333   [(set_attr "type" "mve_move")
 334 ])
 335
 336 ;;
 337 ;; [vcvtbq_f32_f16])
 338 ;;
 339 (define_insn "mve_vcvtbq_f32_f16v4sf"
 340   [
 341    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 342         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 343          VCVTBQ_F32_F16))
 344   ]
 345   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 346   "vcvtb.f32.f16 %q0, %q1"
 347   [(set_attr "type" "mve_move")
 348 ])
 349
 350 ;;
 351 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 352 ;;
 353 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 354   [
 355    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 356         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 357          VCVTQ_TO_F))
 358   ]
 359   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 360   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 361   [(set_attr "type" "mve_move")
 362 ])
 363
 364 ;;
 365 ;; [vrev64q_u, vrev64q_s])
 366 ;;
 367 (define_insn "mve_vrev64q_<supf><mode>"
 368   [
 369    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 370         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 371          VREV64Q))
 372   ]
 373   "TARGET_HAVE_MVE"
 374   "vrev64.%#<V_sz_elem> %q0, %q1"
 375   [(set_attr "type" "mve_move")
 376 ])
 377
 378 ;;
 379 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 380 ;;
 381 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 382   [
 383    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 384         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 385          VCVTQ_FROM_F))
 386   ]
 387   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 388   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 389   [(set_attr "type" "mve_move")
 390 ])
 391 ;; [vqnegq_s])
 392 ;;
 393 (define_insn "mve_vqnegq_s<mode>"
 394   [
 395    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 396         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 397          VQNEGQ_S))
 398   ]
 399   "TARGET_HAVE_MVE"
 400   "vqneg.s%#<V_sz_elem> %q0, %q1"
 401   [(set_attr "type" "mve_move")
 402 ])
 403
 404 ;;
 405 ;; [vqabsq_s])
 406 ;;
 407 (define_insn "mve_vqabsq_s<mode>"
 408   [
 409    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 410         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 411          VQABSQ_S))
 412   ]
 413   "TARGET_HAVE_MVE"
 414   "vqabs.s%#<V_sz_elem> %q0, %q1"
 415   [(set_attr "type" "mve_move")
 416 ])
 417
 418 ;;
 419 ;; [vnegq_s])
 420 ;;
 421 (define_insn "mve_vnegq_s<mode>"
 422   [
 423    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 424         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 425   ]
 426   "TARGET_HAVE_MVE"
 427   "vneg.s%#<V_sz_elem>  %q0, %q1"
 428   [(set_attr "type" "mve_move")
 429 ])
 430
 431 ;;
 432 ;; [vmvnq_u, vmvnq_s])
 433 ;;
 434 (define_insn "mve_vmvnq_u<mode>"
 435   [
 436    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 437         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 438   ]
 439   "TARGET_HAVE_MVE"
 440   "vmvn\t%q0, %q1"
 441   [(set_attr "type" "mve_move")
 442 ])
 443 (define_expand "mve_vmvnq_s<mode>"
 444   [
 445    (set (match_operand:MVE_2 0 "s_register_operand")
 446         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 447   ]
 448   "TARGET_HAVE_MVE"
 449 )
 450
 451 ;;
 452 ;; [vdupq_n_u, vdupq_n_s])
 453 ;;
 454 (define_insn "mve_vdupq_n_<supf><mode>"
 455   [
 456    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 457         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 458          VDUPQ_N))
 459   ]
 460   "TARGET_HAVE_MVE"
 461   "vdup.%#<V_sz_elem>   %q0, %1"
 462   [(set_attr "type" "mve_move")
 463 ])
 464
 465 ;;
 466 ;; [vclzq_u, vclzq_s])
 467 ;;
 468 (define_insn "mve_vclzq_<supf><mode>"
 469   [
 470    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 471         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 472          VCLZQ))
 473   ]
 474   "TARGET_HAVE_MVE"
 475   "vclz.i%#<V_sz_elem>  %q0, %q1"
 476   [(set_attr "type" "mve_move")
 477 ])
 478
 479 ;;
 480 ;; [vclsq_s])
 481 ;;
 482 (define_insn "mve_vclsq_s<mode>"
 483   [
 484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 486          VCLSQ_S))
 487   ]
 488   "TARGET_HAVE_MVE"
 489   "vcls.s%#<V_sz_elem>  %q0, %q1"
 490   [(set_attr "type" "mve_move")
 491 ])
 492
 493 ;;
 494 ;; [vaddvq_s, vaddvq_u])
 495 ;;
 496 (define_insn "mve_vaddvq_<supf><mode>"
 497   [
 498    (set (match_operand:SI 0 "s_register_operand" "=Te")
 499         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 500          VADDVQ))
 501   ]
 502   "TARGET_HAVE_MVE"
 503   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 504   [(set_attr "type" "mve_move")
 505 ])
 506
 507 ;;
 508 ;; [vabsq_s])
 509 ;;
 510 (define_insn "mve_vabsq_s<mode>"
 511   [
 512    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 513         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 514          VABSQ_S))
 515   ]
 516   "TARGET_HAVE_MVE"
 517   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 518   [(set_attr "type" "mve_move")
 519 ])
 520
 521 ;;
 522 ;; [vrev32q_u, vrev32q_s])
 523 ;;
 524 (define_insn "mve_vrev32q_<supf><mode>"
 525   [
 526    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 527         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 528          VREV32Q))
 529   ]
 530   "TARGET_HAVE_MVE"
 531   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 532   [(set_attr "type" "mve_move")
 533 ])
 534
 535 ;;
 536 ;; [vmovltq_u, vmovltq_s])
 537 ;;
 538 (define_insn "mve_vmovltq_<supf><mode>"
 539   [
 540    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 541         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 542          VMOVLTQ))
 543   ]
 544   "TARGET_HAVE_MVE"
 545   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 546   [(set_attr "type" "mve_move")
 547 ])
 548
 549 ;;
 550 ;; [vmovlbq_s, vmovlbq_u])
 551 ;;
 552 (define_insn "mve_vmovlbq_<supf><mode>"
 553   [
 554    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 555         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 556          VMOVLBQ))
 557   ]
 558   "TARGET_HAVE_MVE"
 559   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 560   [(set_attr "type" "mve_move")
 561 ])
 562
 563 ;;
 564 ;; [vcvtpq_s, vcvtpq_u])
 565 ;;
 566 (define_insn "mve_vcvtpq_<supf><mode>"
 567   [
 568    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 569         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 570          VCVTPQ))
 571   ]
 572   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 573   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 574   [(set_attr "type" "mve_move")
 575 ])
 576
 577 ;;
 578 ;; [vcvtnq_s, vcvtnq_u])
 579 ;;
 580 (define_insn "mve_vcvtnq_<supf><mode>"
 581   [
 582    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 583         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 584          VCVTNQ))
 585   ]
 586   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 587   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 588   [(set_attr "type" "mve_move")
 589 ])
 590
 591 ;;
 592 ;; [vcvtmq_s, vcvtmq_u])
 593 ;;
 594 (define_insn "mve_vcvtmq_<supf><mode>"
 595   [
 596    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 597         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 598          VCVTMQ))
 599   ]
 600   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 601   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 602   [(set_attr "type" "mve_move")
 603 ])
 604
 605 ;;
 606 ;; [vcvtaq_u, vcvtaq_s])
 607 ;;
 608 (define_insn "mve_vcvtaq_<supf><mode>"
 609   [
 610    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 611         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 612          VCVTAQ))
 613   ]
 614   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 615   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 616   [(set_attr "type" "mve_move")
 617 ])
 618
 619 ;;
 620 ;; [vmvnq_n_u, vmvnq_n_s])
 621 ;;
 622 (define_insn "mve_vmvnq_n_<supf><mode>"
 623   [
 624    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 625         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 626          VMVNQ_N))
 627   ]
 628   "TARGET_HAVE_MVE"
 629   "vmvn.i%#<V_sz_elem>  %q0, %1"
 630   [(set_attr "type" "mve_move")
 631 ])
 632
 633 ;;
 634 ;; [vrev16q_u, vrev16q_s])
 635 ;;
 636 (define_insn "mve_vrev16q_<supf>v16qi"
 637   [
 638    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 639         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 640          VREV16Q))
 641   ]
 642   "TARGET_HAVE_MVE"
 643   "vrev16.8 %q0, %q1"
 644   [(set_attr "type" "mve_move")
 645 ])
 646
 647 ;;
 648 ;; [vaddlvq_s vaddlvq_u])
 649 ;;
 650 (define_insn "mve_vaddlvq_<supf>v4si"
 651   [
 652    (set (match_operand:DI 0 "s_register_operand" "=r")
 653         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 654          VADDLVQ))
 655   ]
 656   "TARGET_HAVE_MVE"
 657   "vaddlv.<supf>32 %Q0, %R0, %q1"
 658   [(set_attr "type" "mve_move")
 659 ])
 660
 661 ;;
 662 ;; [vctp8q vctp16q vctp32q vctp64q])
 663 ;;
 664 (define_insn "mve_vctp<mode1>qhi"
 665   [
 666    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 667         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 668         VCTPQ))
 669   ]
 670   "TARGET_HAVE_MVE"
 671   "vctp.<mode1> %1"
 672   [(set_attr "type" "mve_move")
 673 ])
 674
 675 ;;
 676 ;; [vpnot])
 677 ;;
 678 (define_insn "mve_vpnothi"
 679   [
 680    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 681         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 682          VPNOT))
 683   ]
 684   "TARGET_HAVE_MVE"
 685   "vpnot"
 686   [(set_attr "type" "mve_move")
 687 ])
 688
 689 ;;
 690 ;; [vsubq_n_f])
 691 ;;
 692 (define_insn "mve_vsubq_n_f<mode>"
 693   [
 694    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 695         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 696                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 697          VSUBQ_N_F))
 698   ]
 699   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 700   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 701   [(set_attr "type" "mve_move")
 702 ])
 703
 704 ;;
 705 ;; [vbrsrq_n_f])
 706 ;;
 707 (define_insn "mve_vbrsrq_n_f<mode>"
 708   [
 709    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 710         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 711                        (match_operand:SI 2 "s_register_operand" "r")]
 712          VBRSRQ_N_F))
 713   ]
 714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 715   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 716   [(set_attr "type" "mve_move")
 717 ])
 718
 719 ;;
 720 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 721 ;;
 722 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 723   [
 724    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 725         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 726                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 727          VCVTQ_N_TO_F))
 728   ]
 729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 730   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 731   [(set_attr "type" "mve_move")
 732 ])
 733
 734 ;; [vcreateq_f])
 735 ;;
 736 (define_insn "mve_vcreateq_f<mode>"
 737   [
 738    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 739         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 740                        (match_operand:DI 2 "s_register_operand" "r")]
 741          VCREATEQ_F))
 742   ]
 743   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 744   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 745   [(set_attr "type" "mve_move")
 746    (set_attr "length""8")])
 747
 748 ;;
 749 ;; [vcreateq_u, vcreateq_s])
 750 ;;
 751 (define_insn "mve_vcreateq_<supf><mode>"
 752   [
 753    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 754         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 755                        (match_operand:DI 2 "s_register_operand" "r")]
 756          VCREATEQ))
 757   ]
 758   "TARGET_HAVE_MVE"
 759   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 760   [(set_attr "type" "mve_move")
 761    (set_attr "length""8")])
 762
 763 ;;
 764 ;; [vshrq_n_s, vshrq_n_u])
 765 ;;
 766 (define_insn "mve_vshrq_n_<supf><mode>"
 767   [
 768    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 769         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 770                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 771          VSHRQ_N))
 772   ]
 773   "TARGET_HAVE_MVE"
 774   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 775   [(set_attr "type" "mve_move")
 776 ])
 777
 778 ;;
 779 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 780 ;;
 781 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 782   [
 783    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 784         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 785                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 786          VCVTQ_N_FROM_F))
 787   ]
 788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 789   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 790   [(set_attr "type" "mve_move")
 791 ])
 792
 793 ;;
 794 ;; [vaddlvq_p_s])
 795 ;;
 796 (define_insn "mve_vaddlvq_p_<supf>v4si"
 797   [
 798    (set (match_operand:DI 0 "s_register_operand" "=r")
 799         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 800                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 801          VADDLVQ_P))
 802   ]
 803   "TARGET_HAVE_MVE"
 804   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 805   [(set_attr "type" "mve_move")
 806    (set_attr "length""8")])
 807
 808 ;;
 809 ;; [vcmpneq_u, vcmpneq_s])
 810 ;;
 811 (define_insn "mve_vcmpneq_<supf><mode>"
 812   [
 813    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 814         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 815                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 816          VCMPNEQ))
 817   ]
 818   "TARGET_HAVE_MVE"
 819   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
 820   [(set_attr "type" "mve_move")
 821 ])
 822
 823 ;;
 824 ;; [vshlq_s, vshlq_u])
 825 ;;
 826 (define_insn "mve_vshlq_<supf><mode>"
 827   [
 828    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 829         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 830                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 831          VSHLQ))
 832   ]
 833   "TARGET_HAVE_MVE"
 834   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
 835   [(set_attr "type" "mve_move")
 836 ])
 837
 838 ;;
 839 ;; [vabdq_s, vabdq_u])
 840 ;;
 841 (define_insn "mve_vabdq_<supf><mode>"
 842   [
 843    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 844         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 845                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 846          VABDQ))
 847   ]
 848   "TARGET_HAVE_MVE"
 849   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 850   [(set_attr "type" "mve_move")
 851 ])
 852
 853 ;;
 854 ;; [vaddq_n_s, vaddq_n_u])
 855 ;;
 856 (define_insn "mve_vaddq_n_<supf><mode>"
 857   [
 858    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 859         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 860                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 861          VADDQ_N))
 862   ]
 863   "TARGET_HAVE_MVE"
 864   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 865   [(set_attr "type" "mve_move")
 866 ])
 867
 868 ;;
 869 ;; [vaddvaq_s, vaddvaq_u])
 870 ;;
 871 (define_insn "mve_vaddvaq_<supf><mode>"
 872   [
 873    (set (match_operand:SI 0 "s_register_operand" "=Te")
 874         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 875                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 876          VADDVAQ))
 877   ]
 878   "TARGET_HAVE_MVE"
 879   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 880   [(set_attr "type" "mve_move")
 881 ])
 882
 883 ;;
 884 ;; [vaddvq_p_u, vaddvq_p_s])
 885 ;;
 886 (define_insn "mve_vaddvq_p_<supf><mode>"
 887   [
 888    (set (match_operand:SI 0 "s_register_operand" "=Te")
 889         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 890                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 891          VADDVQ_P))
 892   ]
 893   "TARGET_HAVE_MVE"
 894   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 895   [(set_attr "type" "mve_move")
 896    (set_attr "length""8")])
 897
 898 ;;
 899 ;; [vandq_u, vandq_s])
 900 ;;
 901 ;; signed and unsigned versions are the same: define the unsigned
 902 ;; insn, and use an expander for the signed one as we still reference
 903 ;; both names from arm_mve.h.
 904 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 905 (define_insn "mve_vandq_u<mode>"
 906   [
 907    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 908         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 909                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 910   ]
 911   "TARGET_HAVE_MVE"
 912   "@
 913    vand\t%q0, %q1, %q2
 914    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 915   [(set_attr "type" "mve_move")
 916 ])
 917 (define_expand "mve_vandq_s<mode>"
 918   [
 919    (set (match_operand:MVE_2 0 "s_register_operand")
 920         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 921                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 922   ]
 923   "TARGET_HAVE_MVE"
 924 )
 925
 926 ;;
 927 ;; [vbicq_s, vbicq_u])
 928 ;;
 929 (define_insn "mve_vbicq_u<mode>"
 930   [
 931    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 932         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 933                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 934   ]
 935   "TARGET_HAVE_MVE"
 936   "vbic\t%q0, %q1, %q2"
 937   [(set_attr "type" "mve_move")
 938 ])
 939
 940 (define_expand "mve_vbicq_s<mode>"
 941   [
 942    (set (match_operand:MVE_2 0 "s_register_operand")
 943         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 944                    (match_operand:MVE_2 1 "s_register_operand")))
 945   ]
 946   "TARGET_HAVE_MVE"
 947 )
 948
 949 ;;
 950 ;; [vbrsrq_n_u, vbrsrq_n_s])
 951 ;;
 952 (define_insn "mve_vbrsrq_n_<supf><mode>"
 953   [
 954    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 955         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 956                        (match_operand:SI 2 "s_register_operand" "r")]
 957          VBRSRQ_N))
 958   ]
 959   "TARGET_HAVE_MVE"
 960   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 961   [(set_attr "type" "mve_move")
 962 ])
 963
 964 ;;
 965 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
 966 ;;
 967 (define_insn "mve_vcaddq_rot270_<supf><mode>"
 968   [
 969    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 970         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 971                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 972          VCADDQ_ROT270))
 973   ]
 974   "TARGET_HAVE_MVE"
 975   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
 976   [(set_attr "type" "mve_move")
 977 ])
 978
 979 ;;
 980 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
 981 ;;
 982 (define_insn "mve_vcaddq_rot90_<supf><mode>"
 983   [
 984    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 985         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 986                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 987          VCADDQ_ROT90))
 988   ]
 989   "TARGET_HAVE_MVE"
 990   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
 991   [(set_attr "type" "mve_move")
 992 ])
 993
 994 ;;
 995 ;; [vcmpcsq_n_u])
 996 ;;
 997 (define_insn "mve_vcmpcsq_n_u<mode>"
 998   [
 999    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1000         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1001                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1002          VCMPCSQ_N_U))
1003   ]
1004   "TARGET_HAVE_MVE"
1005   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1006   [(set_attr "type" "mve_move")
1007 ])
1008
1009 ;;
1010 ;; [vcmpcsq_u])
1011 ;;
1012 (define_insn "mve_vcmpcsq_u<mode>"
1013   [
1014    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1015         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1016                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1017          VCMPCSQ_U))
1018   ]
1019   "TARGET_HAVE_MVE"
1020   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1021   [(set_attr "type" "mve_move")
1022 ])
1023
1024 ;;
1025 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1026 ;;
1027 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1028   [
1029    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1030         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1031                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1032          VCMPEQQ_N))
1033   ]
1034   "TARGET_HAVE_MVE"
1035   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1036   [(set_attr "type" "mve_move")
1037 ])
1038
1039 ;;
1040 ;; [vcmpeqq_u, vcmpeqq_s])
1041 ;;
1042 (define_insn "mve_vcmpeqq_<supf><mode>"
1043   [
1044    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1045         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1046                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1047          VCMPEQQ))
1048   ]
1049   "TARGET_HAVE_MVE"
1050   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1051   [(set_attr "type" "mve_move")
1052 ])
1053
1054 ;;
1055 ;; [vcmpgeq_n_s])
1056 ;;
1057 (define_insn "mve_vcmpgeq_n_s<mode>"
1058   [
1059    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1060         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1061                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1062          VCMPGEQ_N_S))
1063   ]
1064   "TARGET_HAVE_MVE"
1065   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1066   [(set_attr "type" "mve_move")
1067 ])
1068
1069 ;;
1070 ;; [vcmpgeq_s])
1071 ;;
1072 (define_insn "mve_vcmpgeq_s<mode>"
1073   [
1074    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1075         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1076                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1077          VCMPGEQ_S))
1078   ]
1079   "TARGET_HAVE_MVE"
1080   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1081   [(set_attr "type" "mve_move")
1082 ])
1083
1084 ;;
1085 ;; [vcmpgtq_n_s])
1086 ;;
1087 (define_insn "mve_vcmpgtq_n_s<mode>"
1088   [
1089    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1090         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1091                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1092          VCMPGTQ_N_S))
1093   ]
1094   "TARGET_HAVE_MVE"
1095   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1096   [(set_attr "type" "mve_move")
1097 ])
1098
1099 ;;
1100 ;; [vcmpgtq_s])
1101 ;;
1102 (define_insn "mve_vcmpgtq_s<mode>"
1103   [
1104    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1105         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1106                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1107          VCMPGTQ_S))
1108   ]
1109   "TARGET_HAVE_MVE"
1110   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1111   [(set_attr "type" "mve_move")
1112 ])
1113
1114 ;;
1115 ;; [vcmphiq_n_u])
1116 ;;
1117 (define_insn "mve_vcmphiq_n_u<mode>"
1118   [
1119    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1120         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1121                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1122          VCMPHIQ_N_U))
1123   ]
1124   "TARGET_HAVE_MVE"
1125   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1126   [(set_attr "type" "mve_move")
1127 ])
1128
1129 ;;
1130 ;; [vcmphiq_u])
1131 ;;
1132 (define_insn "mve_vcmphiq_u<mode>"
1133   [
1134    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1135         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1136                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1137          VCMPHIQ_U))
1138   ]
1139   "TARGET_HAVE_MVE"
1140   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1141   [(set_attr "type" "mve_move")
1142 ])
1143
1144 ;;
1145 ;; [vcmpleq_n_s])
1146 ;;
1147 (define_insn "mve_vcmpleq_n_s<mode>"
1148   [
1149    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1150         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1151                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1152          VCMPLEQ_N_S))
1153   ]
1154   "TARGET_HAVE_MVE"
1155   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1156   [(set_attr "type" "mve_move")
1157 ])
1158
1159 ;;
1160 ;; [vcmpleq_s])
1161 ;;
1162 (define_insn "mve_vcmpleq_s<mode>"
1163   [
1164    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1165         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1166                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1167          VCMPLEQ_S))
1168   ]
1169   "TARGET_HAVE_MVE"
1170   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1171   [(set_attr "type" "mve_move")
1172 ])
1173
1174 ;;
1175 ;; [vcmpltq_n_s])
1176 ;;
1177 (define_insn "mve_vcmpltq_n_s<mode>"
1178   [
1179    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1180         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1181                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1182          VCMPLTQ_N_S))
1183   ]
1184   "TARGET_HAVE_MVE"
1185   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1186   [(set_attr "type" "mve_move")
1187 ])
1188
1189 ;;
1190 ;; [vcmpltq_s])
1191 ;;
1192 (define_insn "mve_vcmpltq_s<mode>"
1193   [
1194    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1195         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1196                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1197          VCMPLTQ_S))
1198   ]
1199   "TARGET_HAVE_MVE"
1200   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1201   [(set_attr "type" "mve_move")
1202 ])
1203
1204 ;;
1205 ;; [vcmpneq_n_u, vcmpneq_n_s])
1206 ;;
1207 (define_insn "mve_vcmpneq_n_<supf><mode>"
1208   [
1209    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1210         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1211                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1212          VCMPNEQ_N))
1213   ]
1214   "TARGET_HAVE_MVE"
1215   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1216   [(set_attr "type" "mve_move")
1217 ])
1218
1219 ;;
1220 ;; [veorq_u, veorq_s])
1221 ;;
1222 (define_insn "mve_veorq_u<mode>"
1223   [
1224    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1225         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1226                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1227   ]
1228   "TARGET_HAVE_MVE"
1229   "veor\t%q0, %q1, %q2"
1230   [(set_attr "type" "mve_move")
1231 ])
1232 (define_expand "mve_veorq_s<mode>"
1233   [
1234    (set (match_operand:MVE_2 0 "s_register_operand")
1235         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1236                    (match_operand:MVE_2 2 "s_register_operand")))
1237   ]
1238   "TARGET_HAVE_MVE"
1239 )
1240
1241 ;;
1242 ;; [vhaddq_n_u, vhaddq_n_s])
1243 ;;
1244 (define_insn "mve_vhaddq_n_<supf><mode>"
1245   [
1246    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1247         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1248                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1249          VHADDQ_N))
1250   ]
1251   "TARGET_HAVE_MVE"
1252   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1253   [(set_attr "type" "mve_move")
1254 ])
1255
1256 ;;
1257 ;; [vhaddq_s, vhaddq_u])
1258 ;;
1259 (define_insn "mve_vhaddq_<supf><mode>"
1260   [
1261    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1262         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1263                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1264          VHADDQ))
1265   ]
1266   "TARGET_HAVE_MVE"
1267   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1268   [(set_attr "type" "mve_move")
1269 ])
1270
1271 ;;
1272 ;; [vhcaddq_rot270_s])
1273 ;;
1274 (define_insn "mve_vhcaddq_rot270_s<mode>"
1275   [
1276    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1277         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1278                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1279          VHCADDQ_ROT270_S))
1280   ]
1281   "TARGET_HAVE_MVE"
1282   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1283   [(set_attr "type" "mve_move")
1284 ])
1285
1286 ;;
1287 ;; [vhcaddq_rot90_s])
1288 ;;
1289 (define_insn "mve_vhcaddq_rot90_s<mode>"
1290   [
1291    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1292         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1293                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1294          VHCADDQ_ROT90_S))
1295   ]
1296   "TARGET_HAVE_MVE"
1297   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1298   [(set_attr "type" "mve_move")
1299 ])
1300
1301 ;;
1302 ;; [vhsubq_n_u, vhsubq_n_s])
1303 ;;
1304 (define_insn "mve_vhsubq_n_<supf><mode>"
1305   [
1306    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1307         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1308                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1309          VHSUBQ_N))
1310   ]
1311   "TARGET_HAVE_MVE"
1312   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1313   [(set_attr "type" "mve_move")
1314 ])
1315
1316 ;;
1317 ;; [vhsubq_s, vhsubq_u])
1318 ;;
1319 (define_insn "mve_vhsubq_<supf><mode>"
1320   [
1321    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1322         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1323                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1324          VHSUBQ))
1325   ]
1326   "TARGET_HAVE_MVE"
1327   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1328   [(set_attr "type" "mve_move")
1329 ])
1330
1331 ;;
1332 ;; [vmaxaq_s])
1333 ;;
1334 (define_insn "mve_vmaxaq_s<mode>"
1335   [
1336    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1337         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1338                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1339          VMAXAQ_S))
1340   ]
1341   "TARGET_HAVE_MVE"
1342   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1343   [(set_attr "type" "mve_move")
1344 ])
1345
1346 ;;
1347 ;; [vmaxavq_s])
1348 ;;
1349 (define_insn "mve_vmaxavq_s<mode>"
1350   [
1351    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1352         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1353                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1354          VMAXAVQ_S))
1355   ]
1356   "TARGET_HAVE_MVE"
1357   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1358   [(set_attr "type" "mve_move")
1359 ])
1360
1361 ;;
1362 ;; [vmaxq_u, vmaxq_s])
1363 ;;
1364 (define_insn "mve_vmaxq_s<mode>"
1365   [
1366    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1367         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1368                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1369   ]
1370   "TARGET_HAVE_MVE"
1371   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1372   [(set_attr "type" "mve_move")
1373 ])
1374
1375 (define_insn "mve_vmaxq_u<mode>"
1376   [
1377    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1378         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1379                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1380   ]
1381   "TARGET_HAVE_MVE"
1382   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1383   [(set_attr "type" "mve_move")
1384 ])
1385
1386 ;;
1387 ;; [vmaxvq_u, vmaxvq_s])
1388 ;;
1389 (define_insn "mve_vmaxvq_<supf><mode>"
1390   [
1391    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1392         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1393                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1394          VMAXVQ))
1395   ]
1396   "TARGET_HAVE_MVE"
1397   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1398   [(set_attr "type" "mve_move")
1399 ])
1400
1401 ;;
1402 ;; [vminaq_s])
1403 ;;
1404 (define_insn "mve_vminaq_s<mode>"
1405   [
1406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1407         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1408                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1409          VMINAQ_S))
1410   ]
1411   "TARGET_HAVE_MVE"
1412   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1413   [(set_attr "type" "mve_move")
1414 ])
1415
1416 ;;
1417 ;; [vminavq_s])
1418 ;;
1419 (define_insn "mve_vminavq_s<mode>"
1420   [
1421    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1422         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1423                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1424          VMINAVQ_S))
1425   ]
1426   "TARGET_HAVE_MVE"
1427   "vminav.s%#<V_sz_elem>\t%0, %q2"
1428   [(set_attr "type" "mve_move")
1429 ])
1430
1431 ;;
1432 ;; [vminq_s, vminq_u])
1433 ;;
1434 (define_insn "mve_vminq_s<mode>"
1435   [
1436    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1437         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1438                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1439   ]
1440   "TARGET_HAVE_MVE"
1441   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1442   [(set_attr "type" "mve_move")
1443 ])
1444
1445 (define_insn "mve_vminq_u<mode>"
1446   [
1447    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1448         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1449                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1450   ]
1451   "TARGET_HAVE_MVE"
1452   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1453   [(set_attr "type" "mve_move")
1454 ])
1455
1456 ;;
1457 ;; [vminvq_u, vminvq_s])
1458 ;;
1459 (define_insn "mve_vminvq_<supf><mode>"
1460   [
1461    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1462         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1463                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1464          VMINVQ))
1465   ]
1466   "TARGET_HAVE_MVE"
1467   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1468   [(set_attr "type" "mve_move")
1469 ])
1470
1471 ;;
1472 ;; [vmladavq_u, vmladavq_s])
1473 ;;
1474 (define_insn "mve_vmladavq_<supf><mode>"
1475   [
1476    (set (match_operand:SI 0 "s_register_operand" "=Te")
1477         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1478                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1479          VMLADAVQ))
1480   ]
1481   "TARGET_HAVE_MVE"
1482   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1483   [(set_attr "type" "mve_move")
1484 ])
1485
1486 ;;
1487 ;; [vmladavxq_s])
1488 ;;
1489 (define_insn "mve_vmladavxq_s<mode>"
1490   [
1491    (set (match_operand:SI 0 "s_register_operand" "=Te")
1492         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1493                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1494          VMLADAVXQ_S))
1495   ]
1496   "TARGET_HAVE_MVE"
1497   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1498   [(set_attr "type" "mve_move")
1499 ])
1500
1501 ;;
1502 ;; [vmlsdavq_s])
1503 ;;
1504 (define_insn "mve_vmlsdavq_s<mode>"
1505   [
1506    (set (match_operand:SI 0 "s_register_operand" "=Te")
1507         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1508                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1509          VMLSDAVQ_S))
1510   ]
1511   "TARGET_HAVE_MVE"
1512   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1513   [(set_attr "type" "mve_move")
1514 ])
1515
1516 ;;
1517 ;; [vmlsdavxq_s])
1518 ;;
1519 (define_insn "mve_vmlsdavxq_s<mode>"
1520   [
1521    (set (match_operand:SI 0 "s_register_operand" "=Te")
1522         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1523                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1524          VMLSDAVXQ_S))
1525   ]
1526   "TARGET_HAVE_MVE"
1527   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1528   [(set_attr "type" "mve_move")
1529 ])
1530
1531 ;;
1532 ;; [vmulhq_s, vmulhq_u])
1533 ;;
1534 (define_insn "mve_vmulhq_<supf><mode>"
1535   [
1536    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1537         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1538                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1539          VMULHQ))
1540   ]
1541   "TARGET_HAVE_MVE"
1542   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1543   [(set_attr "type" "mve_move")
1544 ])
1545
1546 ;;
1547 ;; [vmullbq_int_u, vmullbq_int_s])
1548 ;;
1549 (define_insn "mve_vmullbq_int_<supf><mode>"
1550   [
1551    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1552         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1553                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1554          VMULLBQ_INT))
1555   ]
1556   "TARGET_HAVE_MVE"
1557   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1558   [(set_attr "type" "mve_move")
1559 ])
1560
1561 ;;
1562 ;; [vmulltq_int_u, vmulltq_int_s])
1563 ;;
1564 (define_insn "mve_vmulltq_int_<supf><mode>"
1565   [
1566    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1567         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1568                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1569          VMULLTQ_INT))
1570   ]
1571   "TARGET_HAVE_MVE"
1572   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1573   [(set_attr "type" "mve_move")
1574 ])
1575
1576 ;;
1577 ;; [vmulq_n_u, vmulq_n_s])
1578 ;;
1579 (define_insn "mve_vmulq_n_<supf><mode>"
1580   [
1581    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1582         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1583                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1584          VMULQ_N))
1585   ]
1586   "TARGET_HAVE_MVE"
1587   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1588   [(set_attr "type" "mve_move")
1589 ])
1590
1591 ;;
1592 ;; [vmulq_u, vmulq_s])
1593 ;;
1594 (define_insn "mve_vmulq_<supf><mode>"
1595   [
1596    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1597         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1598                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1599          VMULQ))
1600   ]
1601   "TARGET_HAVE_MVE"
1602   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1603   [(set_attr "type" "mve_move")
1604 ])
1605
1606 (define_insn "mve_vmulq<mode>"
1607   [
1608    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1609         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1610                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1611   ]
1612   "TARGET_HAVE_MVE"
1613   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1614   [(set_attr "type" "mve_move")
1615 ])
1616
1617 ;;
1618 ;; [vornq_u, vornq_s])
1619 ;;
1620 (define_insn "mve_vornq_<supf><mode>"
1621   [
1622    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1623         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1624                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1625          VORNQ))
1626   ]
1627   "TARGET_HAVE_MVE"
1628   "vorn %q0, %q1, %q2"
1629   [(set_attr "type" "mve_move")
1630 ])
1631
1632 ;;
1633 ;; [vorrq_s, vorrq_u])
1634 ;;
1635 ;; signed and unsigned versions are the same: define the unsigned
1636 ;; insn, and use an expander for the signed one as we still reference
1637 ;; both names from arm_mve.h.
1638 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1639 (define_insn "mve_vorrq_s<mode>"
1640   [
1641    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1642         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1643                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1644   ]
1645   "TARGET_HAVE_MVE"
1646   "@
1647    vorr\t%q0, %q1, %q2
1648    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1649   [(set_attr "type" "mve_move")
1650 ])
1651 (define_expand "mve_vorrq_u<mode>"
1652   [
1653    (set (match_operand:MVE_2 0 "s_register_operand")
1654         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1655                    (match_operand:MVE_2 2 "neon_logic_op2")))
1656   ]
1657   "TARGET_HAVE_MVE"
1658 )
1659
1660 ;;
1661 ;; [vqaddq_n_s, vqaddq_n_u])
1662 ;;
1663 (define_insn "mve_vqaddq_n_<supf><mode>"
1664   [
1665    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1666         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1667                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1668          VQADDQ_N))
1669   ]
1670   "TARGET_HAVE_MVE"
1671   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1672   [(set_attr "type" "mve_move")
1673 ])
1674
1675 ;;
1676 ;; [vqaddq_u, vqaddq_s])
1677 ;;
1678 (define_insn "mve_vqaddq_<supf><mode>"
1679   [
1680    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1681         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1682                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1683          VQADDQ))
1684   ]
1685   "TARGET_HAVE_MVE"
1686   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1687   [(set_attr "type" "mve_move")
1688 ])
1689
1690 ;;
1691 ;; [vqdmulhq_n_s])
1692 ;;
1693 (define_insn "mve_vqdmulhq_n_s<mode>"
1694   [
1695    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1696         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1697                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1698          VQDMULHQ_N_S))
1699   ]
1700   "TARGET_HAVE_MVE"
1701   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1702   [(set_attr "type" "mve_move")
1703 ])
1704
1705 ;;
1706 ;; [vqdmulhq_s])
1707 ;;
1708 (define_insn "mve_vqdmulhq_s<mode>"
1709   [
1710    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1711         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1712                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1713          VQDMULHQ_S))
1714   ]
1715   "TARGET_HAVE_MVE"
1716   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1717   [(set_attr "type" "mve_move")
1718 ])
1719
1720 ;;
1721 ;; [vqrdmulhq_n_s])
1722 ;;
1723 (define_insn "mve_vqrdmulhq_n_s<mode>"
1724   [
1725    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1726         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1727                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1728          VQRDMULHQ_N_S))
1729   ]
1730   "TARGET_HAVE_MVE"
1731   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1732   [(set_attr "type" "mve_move")
1733 ])
1734
1735 ;;
1736 ;; [vqrdmulhq_s])
1737 ;;
1738 (define_insn "mve_vqrdmulhq_s<mode>"
1739   [
1740    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1741         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1742                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1743          VQRDMULHQ_S))
1744   ]
1745   "TARGET_HAVE_MVE"
1746   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1747   [(set_attr "type" "mve_move")
1748 ])
1749
1750 ;;
1751 ;; [vqrshlq_n_s, vqrshlq_n_u])
1752 ;;
1753 (define_insn "mve_vqrshlq_n_<supf><mode>"
1754   [
1755    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1756         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1757                        (match_operand:SI 2 "s_register_operand" "r")]
1758          VQRSHLQ_N))
1759   ]
1760   "TARGET_HAVE_MVE"
1761   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1762   [(set_attr "type" "mve_move")
1763 ])
1764
1765 ;;
1766 ;; [vqrshlq_s, vqrshlq_u])
1767 ;;
1768 (define_insn "mve_vqrshlq_<supf><mode>"
1769   [
1770    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1771         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1772                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1773          VQRSHLQ))
1774   ]
1775   "TARGET_HAVE_MVE"
1776   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1777   [(set_attr "type" "mve_move")
1778 ])
1779
1780 ;;
1781 ;; [vqshlq_n_s, vqshlq_n_u])
1782 ;;
1783 (define_insn "mve_vqshlq_n_<supf><mode>"
1784   [
1785    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1786         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1787                        (match_operand:SI 2 "immediate_operand" "i")]
1788          VQSHLQ_N))
1789   ]
1790   "TARGET_HAVE_MVE"
1791   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1792   [(set_attr "type" "mve_move")
1793 ])
1794
1795 ;;
1796 ;; [vqshlq_r_u, vqshlq_r_s])
1797 ;;
1798 (define_insn "mve_vqshlq_r_<supf><mode>"
1799   [
1800    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1801         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1802                        (match_operand:SI 2 "s_register_operand" "r")]
1803          VQSHLQ_R))
1804   ]
1805   "TARGET_HAVE_MVE"
1806   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1807   [(set_attr "type" "mve_move")
1808 ])
1809
1810 ;;
1811 ;; [vqshlq_s, vqshlq_u])
1812 ;;
1813 (define_insn "mve_vqshlq_<supf><mode>"
1814   [
1815    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1816         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1817                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1818          VQSHLQ))
1819   ]
1820   "TARGET_HAVE_MVE"
1821   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1822   [(set_attr "type" "mve_move")
1823 ])
1824
1825 ;;
1826 ;; [vqshluq_n_s])
1827 ;;
1828 (define_insn "mve_vqshluq_n_s<mode>"
1829   [
1830    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1831         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1832                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1833          VQSHLUQ_N_S))
1834   ]
1835   "TARGET_HAVE_MVE"
1836   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1837   [(set_attr "type" "mve_move")
1838 ])
1839
1840 ;;
1841 ;; [vqsubq_n_s, vqsubq_n_u])
1842 ;;
1843 (define_insn "mve_vqsubq_n_<supf><mode>"
1844   [
1845    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1846         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1847                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1848          VQSUBQ_N))
1849   ]
1850   "TARGET_HAVE_MVE"
1851   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1852   [(set_attr "type" "mve_move")
1853 ])
1854
1855 ;;
1856 ;; [vqsubq_u, vqsubq_s])
1857 ;;
1858 (define_insn "mve_vqsubq_<supf><mode>"
1859   [
1860    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1861         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1862                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1863          VQSUBQ))
1864   ]
1865   "TARGET_HAVE_MVE"
1866   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1867   [(set_attr "type" "mve_move")
1868 ])
1869
1870 ;;
1871 ;; [vrhaddq_s, vrhaddq_u])
1872 ;;
1873 (define_insn "mve_vrhaddq_<supf><mode>"
1874   [
1875    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1876         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1877                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1878          VRHADDQ))
1879   ]
1880   "TARGET_HAVE_MVE"
1881   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1882   [(set_attr "type" "mve_move")
1883 ])
1884
1885 ;;
1886 ;; [vrmulhq_s, vrmulhq_u])
1887 ;;
1888 (define_insn "mve_vrmulhq_<supf><mode>"
1889   [
1890    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1891         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1892                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1893          VRMULHQ))
1894   ]
1895   "TARGET_HAVE_MVE"
1896   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1897   [(set_attr "type" "mve_move")
1898 ])
1899
1900 ;;
1901 ;; [vrshlq_n_u, vrshlq_n_s])
1902 ;;
1903 (define_insn "mve_vrshlq_n_<supf><mode>"
1904   [
1905    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1906         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1907                        (match_operand:SI 2 "s_register_operand" "r")]
1908          VRSHLQ_N))
1909   ]
1910   "TARGET_HAVE_MVE"
1911   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1912   [(set_attr "type" "mve_move")
1913 ])
1914
1915 ;;
1916 ;; [vrshlq_s, vrshlq_u])
1917 ;;
1918 (define_insn "mve_vrshlq_<supf><mode>"
1919   [
1920    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1921         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1922                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1923          VRSHLQ))
1924   ]
1925   "TARGET_HAVE_MVE"
1926   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1927   [(set_attr "type" "mve_move")
1928 ])
1929
1930 ;;
1931 ;; [vrshrq_n_s, vrshrq_n_u])
1932 ;;
1933 (define_insn "mve_vrshrq_n_<supf><mode>"
1934   [
1935    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1936         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1937                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1938          VRSHRQ_N))
1939   ]
1940   "TARGET_HAVE_MVE"
1941   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1942   [(set_attr "type" "mve_move")
1943 ])
1944
1945 ;;
1946 ;; [vshlq_n_u, vshlq_n_s])
1947 ;;
1948 (define_insn "mve_vshlq_n_<supf><mode>"
1949   [
1950    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1951         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1952                        (match_operand:SI 2 "immediate_operand" "i")]
1953          VSHLQ_N))
1954   ]
1955   "TARGET_HAVE_MVE"
1956   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1957   [(set_attr "type" "mve_move")
1958 ])
1959
1960 ;;
1961 ;; [vshlq_r_s, vshlq_r_u])
1962 ;;
1963 (define_insn "mve_vshlq_r_<supf><mode>"
1964   [
1965    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1966         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1967                        (match_operand:SI 2 "s_register_operand" "r")]
1968          VSHLQ_R))
1969   ]
1970   "TARGET_HAVE_MVE"
1971   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1972   [(set_attr "type" "mve_move")
1973 ])
1974
1975 ;;
1976 ;; [vsubq_n_s, vsubq_n_u])
1977 ;;
1978 (define_insn "mve_vsubq_n_<supf><mode>"
1979   [
1980    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1981         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1982                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1983          VSUBQ_N))
1984   ]
1985   "TARGET_HAVE_MVE"
1986   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1987   [(set_attr "type" "mve_move")
1988 ])
1989
1990 ;;
1991 ;; [vsubq_s, vsubq_u])
1992 ;;
1993 (define_insn "mve_vsubq_<supf><mode>"
1994   [
1995    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1996         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1997                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1998          VSUBQ))
1999   ]
2000   "TARGET_HAVE_MVE"
2001   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2002   [(set_attr "type" "mve_move")
2003 ])
2004
2005 (define_insn "mve_vsubq<mode>"
2006   [
2007    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2008         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
2009                      (match_operand:MVE_2 2 "s_register_operand" "w")))
2010   ]
2011   "TARGET_HAVE_MVE"
2012   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2013   [(set_attr "type" "mve_move")
2014 ])
2015
2016 ;;
2017 ;; [vabdq_f])
2018 ;;
2019 (define_insn "mve_vabdq_f<mode>"
2020   [
2021    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2022         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2023                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2024          VABDQ_F))
2025   ]
2026   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2027   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2028   [(set_attr "type" "mve_move")
2029 ])
2030
2031 ;;
2032 ;; [vaddlvaq_s vaddlvaq_u])
2033 ;;
2034 (define_insn "mve_vaddlvaq_<supf>v4si"
2035   [
2036    (set (match_operand:DI 0 "s_register_operand" "=r")
2037         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2038                     (match_operand:V4SI 2 "s_register_operand" "w")]
2039          VADDLVAQ))
2040   ]
2041   "TARGET_HAVE_MVE"
2042   "vaddlva.<supf>32 %Q0, %R0, %q2"
2043   [(set_attr "type" "mve_move")
2044 ])
2045
2046 ;;
2047 ;; [vaddq_n_f])
2048 ;;
2049 (define_insn "mve_vaddq_n_f<mode>"
2050   [
2051    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2052         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2053                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2054          VADDQ_N_F))
2055   ]
2056   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2057   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2058   [(set_attr "type" "mve_move")
2059 ])
2060
2061 ;;
2062 ;; [vandq_f])
2063 ;;
2064 (define_insn "mve_vandq_f<mode>"
2065   [
2066    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2067         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2068                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2069   ]
2070   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2071   "vand %q0, %q1, %q2"
2072   [(set_attr "type" "mve_move")
2073 ])
2074
2075 ;;
2076 ;; [vbicq_f])
2077 ;;
2078 (define_insn "mve_vbicq_f<mode>"
2079   [
2080    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2081         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
2082                               (match_operand:MVE_0 2 "s_register_operand" "w")))
2083   ]
2084   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2085   "vbic %q0, %q1, %q2"
2086   [(set_attr "type" "mve_move")
2087 ])
2088
2089 ;;
2090 ;; [vbicq_n_s, vbicq_n_u])
2091 ;;
2092 (define_insn "mve_vbicq_n_<supf><mode>"
2093   [
2094    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2095         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2096                        (match_operand:SI 2 "immediate_operand" "i")]
2097          VBICQ_N))
2098   ]
2099   "TARGET_HAVE_MVE"
2100   "vbic.i%#<V_sz_elem>  %q0, %2"
2101   [(set_attr "type" "mve_move")
2102 ])
2103
2104 ;;
2105 ;; [vcaddq_rot270_f])
2106 ;;
2107 (define_insn "mve_vcaddq_rot270_f<mode>"
2108   [
2109    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2110         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2111                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2112          VCADDQ_ROT270_F))
2113   ]
2114   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2115   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2116   [(set_attr "type" "mve_move")
2117 ])
2118
2119 ;;
2120 ;; [vcaddq_rot90_f])
2121 ;;
2122 (define_insn "mve_vcaddq_rot90_f<mode>"
2123   [
2124    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2125         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2126                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2127          VCADDQ_ROT90_F))
2128   ]
2129   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2130   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2131   [(set_attr "type" "mve_move")
2132 ])
2133
2134 ;;
2135 ;; [vcmpeqq_f])
2136 ;;
2137 (define_insn "mve_vcmpeqq_f<mode>"
2138   [
2139    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2140         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2141                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2142          VCMPEQQ_F))
2143   ]
2144   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2145   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2146   [(set_attr "type" "mve_move")
2147 ])
2148
2149 ;;
2150 ;; [vcmpeqq_n_f])
2151 ;;
2152 (define_insn "mve_vcmpeqq_n_f<mode>"
2153   [
2154    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2155         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2156                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2157          VCMPEQQ_N_F))
2158   ]
2159   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2160   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2161   [(set_attr "type" "mve_move")
2162 ])
2163
2164 ;;
2165 ;; [vcmpgeq_f])
2166 ;;
2167 (define_insn "mve_vcmpgeq_f<mode>"
2168   [
2169    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2170         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2171                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2172          VCMPGEQ_F))
2173   ]
2174   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2175   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2176   [(set_attr "type" "mve_move")
2177 ])
2178
2179 ;;
2180 ;; [vcmpgeq_n_f])
2181 ;;
2182 (define_insn "mve_vcmpgeq_n_f<mode>"
2183   [
2184    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2185         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2186                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2187          VCMPGEQ_N_F))
2188   ]
2189   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2190   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2191   [(set_attr "type" "mve_move")
2192 ])
2193
2194 ;;
2195 ;; [vcmpgtq_f])
2196 ;;
2197 (define_insn "mve_vcmpgtq_f<mode>"
2198   [
2199    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2200         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2201                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2202          VCMPGTQ_F))
2203   ]
2204   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2205   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2206   [(set_attr "type" "mve_move")
2207 ])
2208
2209 ;;
2210 ;; [vcmpgtq_n_f])
2211 ;;
2212 (define_insn "mve_vcmpgtq_n_f<mode>"
2213   [
2214    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2215         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2216                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2217          VCMPGTQ_N_F))
2218   ]
2219   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2220   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2221   [(set_attr "type" "mve_move")
2222 ])
2223
2224 ;;
2225 ;; [vcmpleq_f])
2226 ;;
2227 (define_insn "mve_vcmpleq_f<mode>"
2228   [
2229    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2230         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2231                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2232          VCMPLEQ_F))
2233   ]
2234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2235   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2236   [(set_attr "type" "mve_move")
2237 ])
2238
2239 ;;
2240 ;; [vcmpleq_n_f])
2241 ;;
2242 (define_insn "mve_vcmpleq_n_f<mode>"
2243   [
2244    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2245         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2246                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2247          VCMPLEQ_N_F))
2248   ]
2249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2250   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2251   [(set_attr "type" "mve_move")
2252 ])
2253
2254 ;;
2255 ;; [vcmpltq_f])
2256 ;;
2257 (define_insn "mve_vcmpltq_f<mode>"
2258   [
2259    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2260         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2261                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2262          VCMPLTQ_F))
2263   ]
2264   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2265   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2266   [(set_attr "type" "mve_move")
2267 ])
2268
2269 ;;
2270 ;; [vcmpltq_n_f])
2271 ;;
2272 (define_insn "mve_vcmpltq_n_f<mode>"
2273   [
2274    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2275         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2276                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2277          VCMPLTQ_N_F))
2278   ]
2279   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2280   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2281   [(set_attr "type" "mve_move")
2282 ])
2283
2284 ;;
2285 ;; [vcmpneq_f])
2286 ;;
2287 (define_insn "mve_vcmpneq_f<mode>"
2288   [
2289    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2290         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2291                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2292          VCMPNEQ_F))
2293   ]
2294   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2295   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2296   [(set_attr "type" "mve_move")
2297 ])
2298
2299 ;;
2300 ;; [vcmpneq_n_f])
2301 ;;
2302 (define_insn "mve_vcmpneq_n_f<mode>"
2303   [
2304    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2305         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2306                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2307          VCMPNEQ_N_F))
2308   ]
2309   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2310   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2311   [(set_attr "type" "mve_move")
2312 ])
2313
2314 ;;
2315 ;; [vcmulq_f])
2316 ;;
2317 (define_insn "mve_vcmulq_f<mode>"
2318   [
2319    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2320         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2321                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2322          VCMULQ_F))
2323   ]
2324   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2325   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2326   [(set_attr "type" "mve_move")
2327 ])
2328
2329 ;;
2330 ;; [vcmulq_rot180_f])
2331 ;;
2332 (define_insn "mve_vcmulq_rot180_f<mode>"
2333   [
2334    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2335         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2336                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2337          VCMULQ_ROT180_F))
2338   ]
2339   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2340   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2341   [(set_attr "type" "mve_move")
2342 ])
2343
2344 ;;
2345 ;; [vcmulq_rot270_f])
2346 ;;
2347 (define_insn "mve_vcmulq_rot270_f<mode>"
2348   [
2349    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2350         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2351                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2352          VCMULQ_ROT270_F))
2353   ]
2354   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2355   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2356   [(set_attr "type" "mve_move")
2357 ])
2358
2359 ;;
2360 ;; [vcmulq_rot90_f])
2361 ;;
2362 (define_insn "mve_vcmulq_rot90_f<mode>"
2363   [
2364    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2365         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2366                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2367          VCMULQ_ROT90_F))
2368   ]
2369   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2370   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2371   [(set_attr "type" "mve_move")
2372 ])
2373
2374 ;;
2375 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2376 ;;
2377 (define_insn "mve_vctp<mode1>q_mhi"
2378   [
2379    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2380         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2381                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2382          VCTPQ_M))
2383   ]
2384   "TARGET_HAVE_MVE"
2385   "vpst\;vctpt.<mode1> %1"
2386   [(set_attr "type" "mve_move")
2387    (set_attr "length""8")])
2388
2389 ;;
2390 ;; [vcvtbq_f16_f32])
2391 ;;
2392 (define_insn "mve_vcvtbq_f16_f32v8hf"
2393   [
2394    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2395         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2396                       (match_operand:V4SF 2 "s_register_operand" "w")]
2397          VCVTBQ_F16_F32))
2398   ]
2399   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2400   "vcvtb.f16.f32 %q0, %q2"
2401   [(set_attr "type" "mve_move")
2402 ])
2403
2404 ;;
2405 ;; [vcvttq_f16_f32])
2406 ;;
2407 (define_insn "mve_vcvttq_f16_f32v8hf"
2408   [
2409    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2410         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2411                       (match_operand:V4SF 2 "s_register_operand" "w")]
2412          VCVTTQ_F16_F32))
2413   ]
2414   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2415   "vcvtt.f16.f32 %q0, %q2"
2416   [(set_attr "type" "mve_move")
2417 ])
2418
2419 ;;
2420 ;; [veorq_f])
2421 ;;
2422 (define_insn "mve_veorq_f<mode>"
2423   [
2424    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2425         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2426                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2427   ]
2428   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2429   "veor %q0, %q1, %q2"
2430   [(set_attr "type" "mve_move")
2431 ])
2432
2433 ;;
2434 ;; [vmaxnmaq_f])
2435 ;;
2436 (define_insn "mve_vmaxnmaq_f<mode>"
2437   [
2438    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2439         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2440                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2441          VMAXNMAQ_F))
2442   ]
2443   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2444   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2445   [(set_attr "type" "mve_move")
2446 ])
2447
2448 ;;
2449 ;; [vmaxnmavq_f])
2450 ;;
2451 (define_insn "mve_vmaxnmavq_f<mode>"
2452   [
2453    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2454         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2455                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2456          VMAXNMAVQ_F))
2457   ]
2458   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2459   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2460   [(set_attr "type" "mve_move")
2461 ])
2462
2463 ;;
2464 ;; [vmaxnmq_f])
2465 ;;
2466 (define_insn "mve_vmaxnmq_f<mode>"
2467   [
2468    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2469         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2470                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2471   ]
2472   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2473   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2474   [(set_attr "type" "mve_move")
2475 ])
2476
2477 ;;
2478 ;; [vmaxnmvq_f])
2479 ;;
2480 (define_insn "mve_vmaxnmvq_f<mode>"
2481   [
2482    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2483         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2484                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2485          VMAXNMVQ_F))
2486   ]
2487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2488   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2489   [(set_attr "type" "mve_move")
2490 ])
2491
2492 ;;
2493 ;; [vminnmaq_f])
2494 ;;
2495 (define_insn "mve_vminnmaq_f<mode>"
2496   [
2497    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2498         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2499                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2500          VMINNMAQ_F))
2501   ]
2502   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2503   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2504   [(set_attr "type" "mve_move")
2505 ])
2506
2507 ;;
2508 ;; [vminnmavq_f])
2509 ;;
2510 (define_insn "mve_vminnmavq_f<mode>"
2511   [
2512    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2513         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2514                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2515          VMINNMAVQ_F))
2516   ]
2517   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2518   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2519   [(set_attr "type" "mve_move")
2520 ])
2521
2522 ;;
2523 ;; [vminnmq_f])
2524 ;;
2525 (define_insn "mve_vminnmq_f<mode>"
2526   [
2527    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2528         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2529                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2530   ]
2531   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2532   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2533   [(set_attr "type" "mve_move")
2534 ])
2535
2536 ;;
2537 ;; [vminnmvq_f])
2538 ;;
2539 (define_insn "mve_vminnmvq_f<mode>"
2540   [
2541    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2542         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2543                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2544          VMINNMVQ_F))
2545   ]
2546   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2547   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2548   [(set_attr "type" "mve_move")
2549 ])
2550
2551 ;;
2552 ;; [vmlaldavq_u, vmlaldavq_s])
2553 ;;
2554 (define_insn "mve_vmlaldavq_<supf><mode>"
2555   [
2556    (set (match_operand:DI 0 "s_register_operand" "=r")
2557         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2558                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2559          VMLALDAVQ))
2560   ]
2561   "TARGET_HAVE_MVE"
2562   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2563   [(set_attr "type" "mve_move")
2564 ])
2565
2566 ;;
2567 ;; [vmlaldavxq_s])
2568 ;;
2569 (define_insn "mve_vmlaldavxq_s<mode>"
2570   [
2571    (set (match_operand:DI 0 "s_register_operand" "=r")
2572         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2573                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2574          VMLALDAVXQ_S))
2575   ]
2576   "TARGET_HAVE_MVE"
2577   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2578   [(set_attr "type" "mve_move")
2579 ])
2580
2581 ;;
2582 ;; [vmlsldavq_s])
2583 ;;
2584 (define_insn "mve_vmlsldavq_s<mode>"
2585   [
2586    (set (match_operand:DI 0 "s_register_operand" "=r")
2587         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2588                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2589          VMLSLDAVQ_S))
2590   ]
2591   "TARGET_HAVE_MVE"
2592   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2593   [(set_attr "type" "mve_move")
2594 ])
2595
2596 ;;
2597 ;; [vmlsldavxq_s])
2598 ;;
2599 (define_insn "mve_vmlsldavxq_s<mode>"
2600   [
2601    (set (match_operand:DI 0 "s_register_operand" "=r")
2602         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2603                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2604          VMLSLDAVXQ_S))
2605   ]
2606   "TARGET_HAVE_MVE"
2607   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2608   [(set_attr "type" "mve_move")
2609 ])
2610
2611 ;;
2612 ;; [vmovnbq_u, vmovnbq_s])
2613 ;;
2614 (define_insn "mve_vmovnbq_<supf><mode>"
2615   [
2616    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2617         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2618                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2619          VMOVNBQ))
2620   ]
2621   "TARGET_HAVE_MVE"
2622   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2623   [(set_attr "type" "mve_move")
2624 ])
2625
2626 ;;
2627 ;; [vmovntq_s, vmovntq_u])
2628 ;;
2629 (define_insn "mve_vmovntq_<supf><mode>"
2630   [
2631    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2632         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2633                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2634          VMOVNTQ))
2635   ]
2636   "TARGET_HAVE_MVE"
2637   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2638   [(set_attr "type" "mve_move")
2639 ])
2640
2641 ;;
2642 ;; [vmulq_f])
2643 ;;
2644 (define_insn "mve_vmulq_f<mode>"
2645   [
2646    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2647         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2648                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2649   ]
2650   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2651   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2652   [(set_attr "type" "mve_move")
2653 ])
2654
2655 ;;
2656 ;; [vmulq_n_f])
2657 ;;
2658 (define_insn "mve_vmulq_n_f<mode>"
2659   [
2660    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2661         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2662                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2663          VMULQ_N_F))
2664   ]
2665   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2666   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2667   [(set_attr "type" "mve_move")
2668 ])
2669
2670 ;;
2671 ;; [vornq_f])
2672 ;;
2673 (define_insn "mve_vornq_f<mode>"
2674   [
2675    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2676         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2677                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2678          VORNQ_F))
2679   ]
2680   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2681   "vorn %q0, %q1, %q2"
2682   [(set_attr "type" "mve_move")
2683 ])
2684
2685 ;;
2686 ;; [vorrq_f])
2687 ;;
2688 (define_insn "mve_vorrq_f<mode>"
2689   [
2690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2691         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2692                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2693   ]
2694   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2695   "vorr %q0, %q1, %q2"
2696   [(set_attr "type" "mve_move")
2697 ])
2698
2699 ;;
2700 ;; [vorrq_n_u, vorrq_n_s])
2701 ;;
2702 (define_insn "mve_vorrq_n_<supf><mode>"
2703   [
2704    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2705         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2706                        (match_operand:SI 2 "immediate_operand" "i")]
2707          VORRQ_N))
2708   ]
2709   "TARGET_HAVE_MVE"
2710   "vorr.i%#<V_sz_elem>  %q0, %2"
2711   [(set_attr "type" "mve_move")
2712 ])
2713
2714 ;;
2715 ;; [vqdmullbq_n_s])
2716 ;;
2717 (define_insn "mve_vqdmullbq_n_s<mode>"
2718   [
2719    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2720         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2721                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2722          VQDMULLBQ_N_S))
2723   ]
2724   "TARGET_HAVE_MVE"
2725   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2726   [(set_attr "type" "mve_move")
2727 ])
2728
2729 ;;
2730 ;; [vqdmullbq_s])
2731 ;;
2732 (define_insn "mve_vqdmullbq_s<mode>"
2733   [
2734    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2735         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2736                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2737          VQDMULLBQ_S))
2738   ]
2739   "TARGET_HAVE_MVE"
2740   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2741   [(set_attr "type" "mve_move")
2742 ])
2743
2744 ;;
2745 ;; [vqdmulltq_n_s])
2746 ;;
2747 (define_insn "mve_vqdmulltq_n_s<mode>"
2748   [
2749    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2750         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2751                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2752          VQDMULLTQ_N_S))
2753   ]
2754   "TARGET_HAVE_MVE"
2755   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2756   [(set_attr "type" "mve_move")
2757 ])
2758
2759 ;;
2760 ;; [vqdmulltq_s])
2761 ;;
2762 (define_insn "mve_vqdmulltq_s<mode>"
2763   [
2764    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2765         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2766                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2767          VQDMULLTQ_S))
2768   ]
2769   "TARGET_HAVE_MVE"
2770   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2771   [(set_attr "type" "mve_move")
2772 ])
2773
2774 ;;
2775 ;; [vqmovnbq_u, vqmovnbq_s])
2776 ;;
2777 (define_insn "mve_vqmovnbq_<supf><mode>"
2778   [
2779    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2780         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2781                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2782          VQMOVNBQ))
2783   ]
2784   "TARGET_HAVE_MVE"
2785   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2786   [(set_attr "type" "mve_move")
2787 ])
2788
2789 ;;
2790 ;; [vqmovntq_u, vqmovntq_s])
2791 ;;
2792 (define_insn "mve_vqmovntq_<supf><mode>"
2793   [
2794    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2795         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2796                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2797          VQMOVNTQ))
2798   ]
2799   "TARGET_HAVE_MVE"
2800   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2801   [(set_attr "type" "mve_move")
2802 ])
2803
2804 ;;
2805 ;; [vqmovunbq_s])
2806 ;;
2807 (define_insn "mve_vqmovunbq_s<mode>"
2808   [
2809    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2810         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2811                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2812          VQMOVUNBQ_S))
2813   ]
2814   "TARGET_HAVE_MVE"
2815   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2816   [(set_attr "type" "mve_move")
2817 ])
2818
2819 ;;
2820 ;; [vqmovuntq_s])
2821 ;;
2822 (define_insn "mve_vqmovuntq_s<mode>"
2823   [
2824    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2825         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2826                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2827          VQMOVUNTQ_S))
2828   ]
2829   "TARGET_HAVE_MVE"
2830   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2831   [(set_attr "type" "mve_move")
2832 ])
2833
2834 ;;
2835 ;; [vrmlaldavhxq_s])
2836 ;;
2837 (define_insn "mve_vrmlaldavhxq_sv4si"
2838   [
2839    (set (match_operand:DI 0 "s_register_operand" "=r")
2840         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2841                     (match_operand:V4SI 2 "s_register_operand" "w")]
2842          VRMLALDAVHXQ_S))
2843   ]
2844   "TARGET_HAVE_MVE"
2845   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2846   [(set_attr "type" "mve_move")
2847 ])
2848
2849 ;;
2850 ;; [vrmlsldavhq_s])
2851 ;;
2852 (define_insn "mve_vrmlsldavhq_sv4si"
2853   [
2854    (set (match_operand:DI 0 "s_register_operand" "=r")
2855         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2856                     (match_operand:V4SI 2 "s_register_operand" "w")]
2857          VRMLSLDAVHQ_S))
2858   ]
2859   "TARGET_HAVE_MVE"
2860   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2861   [(set_attr "type" "mve_move")
2862 ])
2863
2864 ;;
2865 ;; [vrmlsldavhxq_s])
2866 ;;
2867 (define_insn "mve_vrmlsldavhxq_sv4si"
2868   [
2869    (set (match_operand:DI 0 "s_register_operand" "=r")
2870         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2871                     (match_operand:V4SI 2 "s_register_operand" "w")]
2872          VRMLSLDAVHXQ_S))
2873   ]
2874   "TARGET_HAVE_MVE"
2875   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2876   [(set_attr "type" "mve_move")
2877 ])
2878
2879 ;;
2880 ;; [vshllbq_n_s, vshllbq_n_u])
2881 ;;
2882 (define_insn "mve_vshllbq_n_<supf><mode>"
2883   [
2884    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2885         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2886                                   (match_operand:SI 2 "immediate_operand" "i")]
2887          VSHLLBQ_N))
2888   ]
2889   "TARGET_HAVE_MVE"
2890   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2891   [(set_attr "type" "mve_move")
2892 ])
2893
2894 ;;
2895 ;; [vshlltq_n_u, vshlltq_n_s])
2896 ;;
2897 (define_insn "mve_vshlltq_n_<supf><mode>"
2898   [
2899    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2900         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2901                                   (match_operand:SI 2 "immediate_operand" "i")]
2902          VSHLLTQ_N))
2903   ]
2904   "TARGET_HAVE_MVE"
2905   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2906   [(set_attr "type" "mve_move")
2907 ])
2908
2909 ;;
2910 ;; [vsubq_f])
2911 ;;
2912 (define_insn "mve_vsubq_f<mode>"
2913   [
2914    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2915         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2916                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2917   ]
2918   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2919   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2920   [(set_attr "type" "mve_move")
2921 ])
2922
2923 ;;
2924 ;; [vmulltq_poly_p])
2925 ;;
2926 (define_insn "mve_vmulltq_poly_p<mode>"
2927   [
2928    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2929         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2930                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2931          VMULLTQ_POLY_P))
2932   ]
2933   "TARGET_HAVE_MVE"
2934   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2935   [(set_attr "type" "mve_move")
2936 ])
2937
2938 ;;
2939 ;; [vmullbq_poly_p])
2940 ;;
2941 (define_insn "mve_vmullbq_poly_p<mode>"
2942   [
2943    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2944         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2945                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2946          VMULLBQ_POLY_P))
2947   ]
2948   "TARGET_HAVE_MVE"
2949   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2950   [(set_attr "type" "mve_move")
2951 ])
2952
2953 ;;
2954 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2955 ;;
2956 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2957   [
2958    (set (match_operand:DI 0 "s_register_operand" "=r")
2959         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2960                     (match_operand:V4SI 2 "s_register_operand" "w")]
2961          VRMLALDAVHQ))
2962   ]
2963   "TARGET_HAVE_MVE"
2964   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2965   [(set_attr "type" "mve_move")
2966 ])
2967
2968 ;;
2969 ;; [vbicq_m_n_s, vbicq_m_n_u])
2970 ;;
2971 (define_insn "mve_vbicq_m_n_<supf><mode>"
2972   [
2973    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2974         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2975                        (match_operand:SI 2 "immediate_operand" "i")
2976                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2977          VBICQ_M_N))
2978   ]
2979   "TARGET_HAVE_MVE"
2980   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2981   [(set_attr "type" "mve_move")
2982    (set_attr "length""8")])
2983 ;;
2984 ;; [vcmpeqq_m_f])
2985 ;;
2986 (define_insn "mve_vcmpeqq_m_f<mode>"
2987   [
2988    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2989         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2990                     (match_operand:MVE_0 2 "s_register_operand" "w")
2991                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2992          VCMPEQQ_M_F))
2993   ]
2994   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2995   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2996   [(set_attr "type" "mve_move")
2997    (set_attr "length""8")])
2998 ;;
2999 ;; [vcvtaq_m_u, vcvtaq_m_s])
3000 ;;
3001 (define_insn "mve_vcvtaq_m_<supf><mode>"
3002   [
3003    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3004         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3005                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3006                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3007          VCVTAQ_M))
3008   ]
3009   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3010   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3011   [(set_attr "type" "mve_move")
3012    (set_attr "length""8")])
3013 ;;
3014 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3015 ;;
3016 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3017   [
3018    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3019         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3020                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3021                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3022          VCVTQ_M_TO_F))
3023   ]
3024   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3025   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3026   [(set_attr "type" "mve_move")
3027    (set_attr "length""8")])
3028 ;;
3029 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3030 ;;
3031 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3032   [
3033    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3034         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3035                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3036                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3037          VQRSHRNBQ_N))
3038   ]
3039   "TARGET_HAVE_MVE"
3040   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3041   [(set_attr "type" "mve_move")
3042 ])
3043 ;;
3044 ;; [vqrshrunbq_n_s])
3045 ;;
3046 (define_insn "mve_vqrshrunbq_n_s<mode>"
3047   [
3048    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3049         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3050                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3051                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3052          VQRSHRUNBQ_N_S))
3053   ]
3054   "TARGET_HAVE_MVE"
3055   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3056   [(set_attr "type" "mve_move")
3057 ])
3058 ;;
3059 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3060 ;;
3061 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3062   [
3063    (set (match_operand:DI 0 "s_register_operand" "=r")
3064         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3065                     (match_operand:V4SI 2 "s_register_operand" "w")
3066                     (match_operand:V4SI 3 "s_register_operand" "w")]
3067          VRMLALDAVHAQ))
3068   ]
3069   "TARGET_HAVE_MVE"
3070   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3071   [(set_attr "type" "mve_move")
3072 ])
3073
3074 ;;
3075 ;; [vabavq_s, vabavq_u])
3076 ;;
3077 (define_insn "mve_vabavq_<supf><mode>"
3078   [
3079    (set (match_operand:SI 0 "s_register_operand" "=r")
3080         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3081                     (match_operand:MVE_2 2 "s_register_operand" "w")
3082                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3083          VABAVQ))
3084   ]
3085   "TARGET_HAVE_MVE"
3086   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3087   [(set_attr "type" "mve_move")
3088 ])
3089
3090 ;;
3091 ;; [vshlcq_u vshlcq_s]
3092 ;;
3093 (define_expand "mve_vshlcq_vec_<supf><mode>"
3094  [(match_operand:MVE_2 0 "s_register_operand")
3095   (match_operand:MVE_2 1 "s_register_operand")
3096   (match_operand:SI 2 "s_register_operand")
3097   (match_operand:SI 3 "mve_imm_32")
3098   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3099  "TARGET_HAVE_MVE"
3100 {
3101   rtx ignore_wb = gen_reg_rtx (SImode);
3102   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3103                                       operands[2], operands[3]));
3104   DONE;
3105 })
3106
3107 (define_expand "mve_vshlcq_carry_<supf><mode>"
3108  [(match_operand:SI 0 "s_register_operand")
3109   (match_operand:MVE_2 1 "s_register_operand")
3110   (match_operand:SI 2 "s_register_operand")
3111   (match_operand:SI 3 "mve_imm_32")
3112   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3113  "TARGET_HAVE_MVE"
3114 {
3115   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3116   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3117                                       operands[2], operands[3]));
3118   DONE;
3119 })
3120
3121 (define_insn "mve_vshlcq_<supf><mode>"
3122  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3123        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3124                       (match_operand:SI 3 "s_register_operand" "1")
3125                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3126         VSHLCQ))
3127   (set (match_operand:SI  1 "s_register_operand" "=r")
3128        (unspec:SI [(match_dup 2)
3129                    (match_dup 3)
3130                    (match_dup 4)]
3131         VSHLCQ))]
3132  "TARGET_HAVE_MVE"
3133  "vshlc %q0, %1, %4")
3134
3135 ;;
3136 ;; [vabsq_m_s])
3137 ;;
3138 (define_insn "mve_vabsq_m_s<mode>"
3139   [
3140    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3141         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3142                        (match_operand:MVE_2 2 "s_register_operand" "w")
3143                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3144          VABSQ_M_S))
3145   ]
3146   "TARGET_HAVE_MVE"
3147   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3148   [(set_attr "type" "mve_move")
3149    (set_attr "length""8")])
3150
3151 ;;
3152 ;; [vaddvaq_p_u, vaddvaq_p_s])
3153 ;;
3154 (define_insn "mve_vaddvaq_p_<supf><mode>"
3155   [
3156    (set (match_operand:SI 0 "s_register_operand" "=Te")
3157         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3158                        (match_operand:MVE_2 2 "s_register_operand" "w")
3159                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3160          VADDVAQ_P))
3161   ]
3162   "TARGET_HAVE_MVE"
3163   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3164   [(set_attr "type" "mve_move")
3165    (set_attr "length""8")])
3166
3167 ;;
3168 ;; [vclsq_m_s])
3169 ;;
3170 (define_insn "mve_vclsq_m_s<mode>"
3171   [
3172    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3173         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3174                        (match_operand:MVE_2 2 "s_register_operand" "w")
3175                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3176          VCLSQ_M_S))
3177   ]
3178   "TARGET_HAVE_MVE"
3179   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3180   [(set_attr "type" "mve_move")
3181    (set_attr "length""8")])
3182
3183 ;;
3184 ;; [vclzq_m_s, vclzq_m_u])
3185 ;;
3186 (define_insn "mve_vclzq_m_<supf><mode>"
3187   [
3188    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3189         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3190                        (match_operand:MVE_2 2 "s_register_operand" "w")
3191                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3192          VCLZQ_M))
3193   ]
3194   "TARGET_HAVE_MVE"
3195   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3196   [(set_attr "type" "mve_move")
3197    (set_attr "length""8")])
3198
3199 ;;
3200 ;; [vcmpcsq_m_n_u])
3201 ;;
3202 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3203   [
3204    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3205         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3206                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3207                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3208          VCMPCSQ_M_N_U))
3209   ]
3210   "TARGET_HAVE_MVE"
3211   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3212   [(set_attr "type" "mve_move")
3213    (set_attr "length""8")])
3214
3215 ;;
3216 ;; [vcmpcsq_m_u])
3217 ;;
3218 (define_insn "mve_vcmpcsq_m_u<mode>"
3219   [
3220    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3221         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3222                        (match_operand:MVE_2 2 "s_register_operand" "w")
3223                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3224          VCMPCSQ_M_U))
3225   ]
3226   "TARGET_HAVE_MVE"
3227   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3228   [(set_attr "type" "mve_move")
3229    (set_attr "length""8")])
3230
3231 ;;
3232 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3233 ;;
3234 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3235   [
3236    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3237         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3238                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3239                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3240          VCMPEQQ_M_N))
3241   ]
3242   "TARGET_HAVE_MVE"
3243   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3244   [(set_attr "type" "mve_move")
3245    (set_attr "length""8")])
3246
3247 ;;
3248 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3249 ;;
3250 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3251   [
3252    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3253         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3254                        (match_operand:MVE_2 2 "s_register_operand" "w")
3255                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3256          VCMPEQQ_M))
3257   ]
3258   "TARGET_HAVE_MVE"
3259   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3260   [(set_attr "type" "mve_move")
3261    (set_attr "length""8")])
3262
3263 ;;
3264 ;; [vcmpgeq_m_n_s])
3265 ;;
3266 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3267   [
3268    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3269         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3270                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3271                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3272          VCMPGEQ_M_N_S))
3273   ]
3274   "TARGET_HAVE_MVE"
3275   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3276   [(set_attr "type" "mve_move")
3277    (set_attr "length""8")])
3278
3279 ;;
3280 ;; [vcmpgeq_m_s])
3281 ;;
3282 (define_insn "mve_vcmpgeq_m_s<mode>"
3283   [
3284    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3285         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3286                        (match_operand:MVE_2 2 "s_register_operand" "w")
3287                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3288          VCMPGEQ_M_S))
3289   ]
3290   "TARGET_HAVE_MVE"
3291   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3292   [(set_attr "type" "mve_move")
3293    (set_attr "length""8")])
3294
3295 ;;
3296 ;; [vcmpgtq_m_n_s])
3297 ;;
3298 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3299   [
3300    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3301         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3302                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3303                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3304          VCMPGTQ_M_N_S))
3305   ]
3306   "TARGET_HAVE_MVE"
3307   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3308   [(set_attr "type" "mve_move")
3309    (set_attr "length""8")])
3310
3311 ;;
3312 ;; [vcmpgtq_m_s])
3313 ;;
3314 (define_insn "mve_vcmpgtq_m_s<mode>"
3315   [
3316    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3317         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3318                        (match_operand:MVE_2 2 "s_register_operand" "w")
3319                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3320          VCMPGTQ_M_S))
3321   ]
3322   "TARGET_HAVE_MVE"
3323   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3324   [(set_attr "type" "mve_move")
3325    (set_attr "length""8")])
3326
3327 ;;
3328 ;; [vcmphiq_m_n_u])
3329 ;;
3330 (define_insn "mve_vcmphiq_m_n_u<mode>"
3331   [
3332    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3333         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3334                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3335                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3336          VCMPHIQ_M_N_U))
3337   ]
3338   "TARGET_HAVE_MVE"
3339   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3340   [(set_attr "type" "mve_move")
3341    (set_attr "length""8")])
3342
3343 ;;
3344 ;; [vcmphiq_m_u])
3345 ;;
3346 (define_insn "mve_vcmphiq_m_u<mode>"
3347   [
3348    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3349         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3350                        (match_operand:MVE_2 2 "s_register_operand" "w")
3351                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3352          VCMPHIQ_M_U))
3353   ]
3354   "TARGET_HAVE_MVE"
3355   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3356   [(set_attr "type" "mve_move")
3357    (set_attr "length""8")])
3358
3359 ;;
3360 ;; [vcmpleq_m_n_s])
3361 ;;
3362 (define_insn "mve_vcmpleq_m_n_s<mode>"
3363   [
3364    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3365         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3366                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3367                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3368          VCMPLEQ_M_N_S))
3369   ]
3370   "TARGET_HAVE_MVE"
3371   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3372   [(set_attr "type" "mve_move")
3373    (set_attr "length""8")])
3374
3375 ;;
3376 ;; [vcmpleq_m_s])
3377 ;;
3378 (define_insn "mve_vcmpleq_m_s<mode>"
3379   [
3380    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3381         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3382                        (match_operand:MVE_2 2 "s_register_operand" "w")
3383                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3384          VCMPLEQ_M_S))
3385   ]
3386   "TARGET_HAVE_MVE"
3387   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3388   [(set_attr "type" "mve_move")
3389    (set_attr "length""8")])
3390
3391 ;;
3392 ;; [vcmpltq_m_n_s])
3393 ;;
3394 (define_insn "mve_vcmpltq_m_n_s<mode>"
3395   [
3396    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3397         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3398                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3399                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3400          VCMPLTQ_M_N_S))
3401   ]
3402   "TARGET_HAVE_MVE"
3403   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3404   [(set_attr "type" "mve_move")
3405    (set_attr "length""8")])
3406
3407 ;;
3408 ;; [vcmpltq_m_s])
3409 ;;
3410 (define_insn "mve_vcmpltq_m_s<mode>"
3411   [
3412    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3413         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3414                        (match_operand:MVE_2 2 "s_register_operand" "w")
3415                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3416          VCMPLTQ_M_S))
3417   ]
3418   "TARGET_HAVE_MVE"
3419   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3420   [(set_attr "type" "mve_move")
3421    (set_attr "length""8")])
3422
3423 ;;
3424 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3425 ;;
3426 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3427   [
3428    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3429         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3430                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3431                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3432          VCMPNEQ_M_N))
3433   ]
3434   "TARGET_HAVE_MVE"
3435   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3436   [(set_attr "type" "mve_move")
3437    (set_attr "length""8")])
3438
3439 ;;
3440 ;; [vcmpneq_m_s, vcmpneq_m_u])
3441 ;;
3442 (define_insn "mve_vcmpneq_m_<supf><mode>"
3443   [
3444    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3445         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3446                        (match_operand:MVE_2 2 "s_register_operand" "w")
3447                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3448          VCMPNEQ_M))
3449   ]
3450   "TARGET_HAVE_MVE"
3451   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3452   [(set_attr "type" "mve_move")
3453    (set_attr "length""8")])
3454
3455 ;;
3456 ;; [vdupq_m_n_s, vdupq_m_n_u])
3457 ;;
3458 (define_insn "mve_vdupq_m_n_<supf><mode>"
3459   [
3460    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3461         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3462                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3463                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3464          VDUPQ_M_N))
3465   ]
3466   "TARGET_HAVE_MVE"
3467   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3468   [(set_attr "type" "mve_move")
3469    (set_attr "length""8")])
3470
3471 ;;
3472 ;; [vmaxaq_m_s])
3473 ;;
3474 (define_insn "mve_vmaxaq_m_s<mode>"
3475   [
3476    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3477         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3478                        (match_operand:MVE_2 2 "s_register_operand" "w")
3479                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3480          VMAXAQ_M_S))
3481   ]
3482   "TARGET_HAVE_MVE"
3483   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3484   [(set_attr "type" "mve_move")
3485    (set_attr "length""8")])
3486
3487 ;;
3488 ;; [vmaxavq_p_s])
3489 ;;
3490 (define_insn "mve_vmaxavq_p_s<mode>"
3491   [
3492    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3493         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3494                        (match_operand:MVE_2 2 "s_register_operand" "w")
3495                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3496          VMAXAVQ_P_S))
3497   ]
3498   "TARGET_HAVE_MVE"
3499   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3500   [(set_attr "type" "mve_move")
3501    (set_attr "length""8")])
3502
3503 ;;
3504 ;; [vmaxvq_p_u, vmaxvq_p_s])
3505 ;;
3506 (define_insn "mve_vmaxvq_p_<supf><mode>"
3507   [
3508    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3509         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3510                        (match_operand:MVE_2 2 "s_register_operand" "w")
3511                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3512          VMAXVQ_P))
3513   ]
3514   "TARGET_HAVE_MVE"
3515   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3516   [(set_attr "type" "mve_move")
3517    (set_attr "length""8")])
3518
3519 ;;
3520 ;; [vminaq_m_s])
3521 ;;
3522 (define_insn "mve_vminaq_m_s<mode>"
3523   [
3524    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3525         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3526                        (match_operand:MVE_2 2 "s_register_operand" "w")
3527                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3528          VMINAQ_M_S))
3529   ]
3530   "TARGET_HAVE_MVE"
3531   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3532   [(set_attr "type" "mve_move")
3533    (set_attr "length""8")])
3534
3535 ;;
3536 ;; [vminavq_p_s])
3537 ;;
3538 (define_insn "mve_vminavq_p_s<mode>"
3539   [
3540    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3541         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3542                        (match_operand:MVE_2 2 "s_register_operand" "w")
3543                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3544          VMINAVQ_P_S))
3545   ]
3546   "TARGET_HAVE_MVE"
3547   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3548   [(set_attr "type" "mve_move")
3549    (set_attr "length""8")])
3550
3551 ;;
3552 ;; [vminvq_p_s, vminvq_p_u])
3553 ;;
3554 (define_insn "mve_vminvq_p_<supf><mode>"
3555   [
3556    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3557         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3558                        (match_operand:MVE_2 2 "s_register_operand" "w")
3559                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3560          VMINVQ_P))
3561   ]
3562   "TARGET_HAVE_MVE"
3563   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3564   [(set_attr "type" "mve_move")
3565    (set_attr "length""8")])
3566
3567 ;;
3568 ;; [vmladavaq_u, vmladavaq_s])
3569 ;;
3570 (define_insn "mve_vmladavaq_<supf><mode>"
3571   [
3572    (set (match_operand:SI 0 "s_register_operand" "=Te")
3573         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3574                        (match_operand:MVE_2 2 "s_register_operand" "w")
3575                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3576          VMLADAVAQ))
3577   ]
3578   "TARGET_HAVE_MVE"
3579   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3580   [(set_attr "type" "mve_move")
3581 ])
3582
3583 ;;
3584 ;; [vmladavq_p_u, vmladavq_p_s])
3585 ;;
3586 (define_insn "mve_vmladavq_p_<supf><mode>"
3587   [
3588    (set (match_operand:SI 0 "s_register_operand" "=Te")
3589         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3590                        (match_operand:MVE_2 2 "s_register_operand" "w")
3591                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3592          VMLADAVQ_P))
3593   ]
3594   "TARGET_HAVE_MVE"
3595   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3596   [(set_attr "type" "mve_move")
3597    (set_attr "length""8")])
3598
3599 ;;
3600 ;; [vmladavxq_p_s])
3601 ;;
3602 (define_insn "mve_vmladavxq_p_s<mode>"
3603   [
3604    (set (match_operand:SI 0 "s_register_operand" "=Te")
3605         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3606                        (match_operand:MVE_2 2 "s_register_operand" "w")
3607                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3608          VMLADAVXQ_P_S))
3609   ]
3610   "TARGET_HAVE_MVE"
3611   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3612   [(set_attr "type" "mve_move")
3613    (set_attr "length""8")])
3614
3615 ;;
3616 ;; [vmlaq_n_u, vmlaq_n_s])
3617 ;;
3618 (define_insn "mve_vmlaq_n_<supf><mode>"
3619   [
3620    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3621         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3622                        (match_operand:MVE_2 2 "s_register_operand" "w")
3623                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3624          VMLAQ_N))
3625   ]
3626   "TARGET_HAVE_MVE"
3627   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3628   [(set_attr "type" "mve_move")
3629 ])
3630
3631 ;;
3632 ;; [vmlasq_n_u, vmlasq_n_s])
3633 ;;
3634 (define_insn "mve_vmlasq_n_<supf><mode>"
3635   [
3636    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3637         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3638                        (match_operand:MVE_2 2 "s_register_operand" "w")
3639                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3640          VMLASQ_N))
3641   ]
3642   "TARGET_HAVE_MVE"
3643   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3644   [(set_attr "type" "mve_move")
3645 ])
3646
3647 ;;
3648 ;; [vmlsdavq_p_s])
3649 ;;
3650 (define_insn "mve_vmlsdavq_p_s<mode>"
3651   [
3652    (set (match_operand:SI 0 "s_register_operand" "=Te")
3653         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3654                        (match_operand:MVE_2 2 "s_register_operand" "w")
3655                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3656          VMLSDAVQ_P_S))
3657   ]
3658   "TARGET_HAVE_MVE"
3659   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3660   [(set_attr "type" "mve_move")
3661    (set_attr "length""8")])
3662
3663 ;;
3664 ;; [vmlsdavxq_p_s])
3665 ;;
3666 (define_insn "mve_vmlsdavxq_p_s<mode>"
3667   [
3668    (set (match_operand:SI 0 "s_register_operand" "=Te")
3669         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3670                        (match_operand:MVE_2 2 "s_register_operand" "w")
3671                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3672          VMLSDAVXQ_P_S))
3673   ]
3674   "TARGET_HAVE_MVE"
3675   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3676   [(set_attr "type" "mve_move")
3677    (set_attr "length""8")])
3678
3679 ;;
3680 ;; [vmvnq_m_s, vmvnq_m_u])
3681 ;;
3682 (define_insn "mve_vmvnq_m_<supf><mode>"
3683   [
3684    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3685         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3686                        (match_operand:MVE_2 2 "s_register_operand" "w")
3687                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3688          VMVNQ_M))
3689   ]
3690   "TARGET_HAVE_MVE"
3691   "vpst\;vmvnt %q0, %q2"
3692   [(set_attr "type" "mve_move")
3693    (set_attr "length""8")])
3694
3695 ;;
3696 ;; [vnegq_m_s])
3697 ;;
3698 (define_insn "mve_vnegq_m_s<mode>"
3699   [
3700    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3701         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3702                        (match_operand:MVE_2 2 "s_register_operand" "w")
3703                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3704          VNEGQ_M_S))
3705   ]
3706   "TARGET_HAVE_MVE"
3707   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3708   [(set_attr "type" "mve_move")
3709    (set_attr "length""8")])
3710
3711 ;;
3712 ;; [vpselq_u, vpselq_s])
3713 ;;
3714 (define_insn "mve_vpselq_<supf><mode>"
3715   [
3716    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3717         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3718                        (match_operand:MVE_1 2 "s_register_operand" "w")
3719                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3720          VPSELQ))
3721   ]
3722   "TARGET_HAVE_MVE"
3723   "vpsel %q0, %q1, %q2"
3724   [(set_attr "type" "mve_move")
3725 ])
3726
3727 ;;
3728 ;; [vqabsq_m_s])
3729 ;;
3730 (define_insn "mve_vqabsq_m_s<mode>"
3731   [
3732    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3733         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3734                        (match_operand:MVE_2 2 "s_register_operand" "w")
3735                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3736          VQABSQ_M_S))
3737   ]
3738   "TARGET_HAVE_MVE"
3739   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3740   [(set_attr "type" "mve_move")
3741    (set_attr "length""8")])
3742
3743 ;;
3744 ;; [vqdmlahq_n_s])
3745 ;;
3746 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3747   [
3748    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3749         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3750                        (match_operand:MVE_2 2 "s_register_operand" "w")
3751                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3752          VQDMLAHQ_N))
3753   ]
3754   "TARGET_HAVE_MVE"
3755   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3756   [(set_attr "type" "mve_move")
3757 ])
3758
3759 ;;
3760 ;; [vqdmlashq_n_s])
3761 ;;
3762 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3763   [
3764    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3765         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3766                        (match_operand:MVE_2 2 "s_register_operand" "w")
3767                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3768          VQDMLASHQ_N))
3769   ]
3770   "TARGET_HAVE_MVE"
3771   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3772   [(set_attr "type" "mve_move")
3773 ])
3774
3775 ;;
3776 ;; [vqnegq_m_s])
3777 ;;
3778 (define_insn "mve_vqnegq_m_s<mode>"
3779   [
3780    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3781         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3782                        (match_operand:MVE_2 2 "s_register_operand" "w")
3783                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3784          VQNEGQ_M_S))
3785   ]
3786   "TARGET_HAVE_MVE"
3787   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3788   [(set_attr "type" "mve_move")
3789    (set_attr "length""8")])
3790
3791 ;;
3792 ;; [vqrdmladhq_s])
3793 ;;
3794 (define_insn "mve_vqrdmladhq_s<mode>"
3795   [
3796    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3797         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3798                        (match_operand:MVE_2 2 "s_register_operand" "w")
3799                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3800          VQRDMLADHQ_S))
3801   ]
3802   "TARGET_HAVE_MVE"
3803   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3804   [(set_attr "type" "mve_move")
3805 ])
3806
3807 ;;
3808 ;; [vqrdmladhxq_s])
3809 ;;
3810 (define_insn "mve_vqrdmladhxq_s<mode>"
3811   [
3812    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3813         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3814                        (match_operand:MVE_2 2 "s_register_operand" "w")
3815                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3816          VQRDMLADHXQ_S))
3817   ]
3818   "TARGET_HAVE_MVE"
3819   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3820   [(set_attr "type" "mve_move")
3821 ])
3822
3823 ;;
3824 ;; [vqrdmlahq_n_s])
3825 ;;
3826 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3827   [
3828    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3829         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3830                        (match_operand:MVE_2 2 "s_register_operand" "w")
3831                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3832          VQRDMLAHQ_N))
3833   ]
3834   "TARGET_HAVE_MVE"
3835   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3836   [(set_attr "type" "mve_move")
3837 ])
3838
3839 ;;
3840 ;; [vqrdmlashq_n_s])
3841 ;;
3842 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3843   [
3844    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3845         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3846                        (match_operand:MVE_2 2 "s_register_operand" "w")
3847                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3848          VQRDMLASHQ_N))
3849   ]
3850   "TARGET_HAVE_MVE"
3851   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3852   [(set_attr "type" "mve_move")
3853 ])
3854
3855 ;;
3856 ;; [vqrdmlsdhq_s])
3857 ;;
3858 (define_insn "mve_vqrdmlsdhq_s<mode>"
3859   [
3860    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3861         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3862                        (match_operand:MVE_2 2 "s_register_operand" "w")
3863                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3864          VQRDMLSDHQ_S))
3865   ]
3866   "TARGET_HAVE_MVE"
3867   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3868   [(set_attr "type" "mve_move")
3869 ])
3870
3871 ;;
3872 ;; [vqrdmlsdhxq_s])
3873 ;;
3874 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3875   [
3876    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3877         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3878                        (match_operand:MVE_2 2 "s_register_operand" "w")
3879                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3880          VQRDMLSDHXQ_S))
3881   ]
3882   "TARGET_HAVE_MVE"
3883   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3884   [(set_attr "type" "mve_move")
3885 ])
3886
3887 ;;
3888 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3889 ;;
3890 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3891   [
3892    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3893         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3894                        (match_operand:SI 2 "s_register_operand" "r")
3895                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3896          VQRSHLQ_M_N))
3897   ]
3898   "TARGET_HAVE_MVE"
3899   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3900   [(set_attr "type" "mve_move")
3901    (set_attr "length""8")])
3902
3903 ;;
3904 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3905 ;;
3906 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3907   [
3908    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3909         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3910                        (match_operand:SI 2 "s_register_operand" "r")
3911                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3912          VQSHLQ_M_R))
3913   ]
3914   "TARGET_HAVE_MVE"
3915   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3916   [(set_attr "type" "mve_move")
3917    (set_attr "length""8")])
3918
3919 ;;
3920 ;; [vrev64q_m_u, vrev64q_m_s])
3921 ;;
3922 (define_insn "mve_vrev64q_m_<supf><mode>"
3923   [
3924    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3925         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3926                        (match_operand:MVE_2 2 "s_register_operand" "w")
3927                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3928          VREV64Q_M))
3929   ]
3930   "TARGET_HAVE_MVE"
3931   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3932   [(set_attr "type" "mve_move")
3933    (set_attr "length""8")])
3934
3935 ;;
3936 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3937 ;;
3938 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3939   [
3940    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3941         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3942                        (match_operand:SI 2 "s_register_operand" "r")
3943                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3944          VRSHLQ_M_N))
3945   ]
3946   "TARGET_HAVE_MVE"
3947   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3948   [(set_attr "type" "mve_move")
3949    (set_attr "length""8")])
3950
3951 ;;
3952 ;; [vshlq_m_r_u, vshlq_m_r_s])
3953 ;;
3954 (define_insn "mve_vshlq_m_r_<supf><mode>"
3955   [
3956    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3957         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3958                        (match_operand:SI 2 "s_register_operand" "r")
3959                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3960          VSHLQ_M_R))
3961   ]
3962   "TARGET_HAVE_MVE"
3963   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3964   [(set_attr "type" "mve_move")
3965    (set_attr "length""8")])
3966
3967 ;;
3968 ;; [vsliq_n_u, vsliq_n_s])
3969 ;;
3970 (define_insn "mve_vsliq_n_<supf><mode>"
3971   [
3972    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3973         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3974                        (match_operand:MVE_2 2 "s_register_operand" "w")
3975                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3976          VSLIQ_N))
3977   ]
3978   "TARGET_HAVE_MVE"
3979   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3980   [(set_attr "type" "mve_move")
3981 ])
3982
3983 ;;
3984 ;; [vsriq_n_u, vsriq_n_s])
3985 ;;
3986 (define_insn "mve_vsriq_n_<supf><mode>"
3987   [
3988    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3989         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3990                        (match_operand:MVE_2 2 "s_register_operand" "w")
3991                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3992          VSRIQ_N))
3993   ]
3994   "TARGET_HAVE_MVE"
3995   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3996   [(set_attr "type" "mve_move")
3997 ])
3998
3999 ;;
4000 ;; [vqdmlsdhxq_s])
4001 ;;
4002 (define_insn "mve_vqdmlsdhxq_s<mode>"
4003   [
4004    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4005         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4006                        (match_operand:MVE_2 2 "s_register_operand" "w")
4007                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4008          VQDMLSDHXQ_S))
4009   ]
4010   "TARGET_HAVE_MVE"
4011   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4012   [(set_attr "type" "mve_move")
4013 ])
4014
4015 ;;
4016 ;; [vqdmlsdhq_s])
4017 ;;
4018 (define_insn "mve_vqdmlsdhq_s<mode>"
4019   [
4020    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4021         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4022                        (match_operand:MVE_2 2 "s_register_operand" "w")
4023                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4024          VQDMLSDHQ_S))
4025   ]
4026   "TARGET_HAVE_MVE"
4027   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4028   [(set_attr "type" "mve_move")
4029 ])
4030
4031 ;;
4032 ;; [vqdmladhxq_s])
4033 ;;
4034 (define_insn "mve_vqdmladhxq_s<mode>"
4035   [
4036    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4037         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4038                        (match_operand:MVE_2 2 "s_register_operand" "w")
4039                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4040          VQDMLADHXQ_S))
4041   ]
4042   "TARGET_HAVE_MVE"
4043   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4044   [(set_attr "type" "mve_move")
4045 ])
4046
4047 ;;
4048 ;; [vqdmladhq_s])
4049 ;;
4050 (define_insn "mve_vqdmladhq_s<mode>"
4051   [
4052    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4053         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4054                        (match_operand:MVE_2 2 "s_register_operand" "w")
4055                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4056          VQDMLADHQ_S))
4057   ]
4058   "TARGET_HAVE_MVE"
4059   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4060   [(set_attr "type" "mve_move")
4061 ])
4062
4063 ;;
4064 ;; [vmlsdavaxq_s])
4065 ;;
4066 (define_insn "mve_vmlsdavaxq_s<mode>"
4067   [
4068    (set (match_operand:SI 0 "s_register_operand" "=Te")
4069         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4070                     (match_operand:MVE_2 2 "s_register_operand" "w")
4071                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4072          VMLSDAVAXQ_S))
4073   ]
4074   "TARGET_HAVE_MVE"
4075   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4076   [(set_attr "type" "mve_move")
4077 ])
4078
4079 ;;
4080 ;; [vmlsdavaq_s])
4081 ;;
4082 (define_insn "mve_vmlsdavaq_s<mode>"
4083   [
4084    (set (match_operand:SI 0 "s_register_operand" "=Te")
4085         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4086                     (match_operand:MVE_2 2 "s_register_operand" "w")
4087                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4088          VMLSDAVAQ_S))
4089   ]
4090   "TARGET_HAVE_MVE"
4091   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4092   [(set_attr "type" "mve_move")
4093 ])
4094
4095 ;;
4096 ;; [vmladavaxq_s])
4097 ;;
4098 (define_insn "mve_vmladavaxq_s<mode>"
4099   [
4100    (set (match_operand:SI 0 "s_register_operand" "=Te")
4101         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4102                     (match_operand:MVE_2 2 "s_register_operand" "w")
4103                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4104          VMLADAVAXQ_S))
4105   ]
4106   "TARGET_HAVE_MVE"
4107   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4108   [(set_attr "type" "mve_move")
4109 ])
4110 ;;
4111 ;; [vabsq_m_f])
4112 ;;
4113 (define_insn "mve_vabsq_m_f<mode>"
4114   [
4115    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4116         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4117                        (match_operand:MVE_0 2 "s_register_operand" "w")
4118                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4119          VABSQ_M_F))
4120   ]
4121   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4122   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4123   [(set_attr "type" "mve_move")
4124    (set_attr "length""8")])
4125
4126 ;;
4127 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4128 ;;
4129 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4130   [
4131    (set (match_operand:DI 0 "s_register_operand" "=r")
4132         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4133                        (match_operand:V4SI 2 "s_register_operand" "w")
4134                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4135          VADDLVAQ_P))
4136   ]
4137   "TARGET_HAVE_MVE"
4138   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4139   [(set_attr "type" "mve_move")
4140    (set_attr "length""8")])
4141 ;;
4142 ;; [vcmlaq_f])
4143 ;;
4144 (define_insn "mve_vcmlaq_f<mode>"
4145   [
4146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4148                        (match_operand:MVE_0 2 "s_register_operand" "w")
4149                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4150          VCMLAQ_F))
4151   ]
4152   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4153   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4154   [(set_attr "type" "mve_move")
4155 ])
4156
4157 ;;
4158 ;; [vcmlaq_rot180_f])
4159 ;;
4160 (define_insn "mve_vcmlaq_rot180_f<mode>"
4161   [
4162    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4163         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4164                        (match_operand:MVE_0 2 "s_register_operand" "w")
4165                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4166          VCMLAQ_ROT180_F))
4167   ]
4168   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4169   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4170   [(set_attr "type" "mve_move")
4171 ])
4172
4173 ;;
4174 ;; [vcmlaq_rot270_f])
4175 ;;
4176 (define_insn "mve_vcmlaq_rot270_f<mode>"
4177   [
4178    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4179         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4180                        (match_operand:MVE_0 2 "s_register_operand" "w")
4181                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4182          VCMLAQ_ROT270_F))
4183   ]
4184   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4185   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4186   [(set_attr "type" "mve_move")
4187 ])
4188
4189 ;;
4190 ;; [vcmlaq_rot90_f])
4191 ;;
4192 (define_insn "mve_vcmlaq_rot90_f<mode>"
4193   [
4194    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4195         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4196                        (match_operand:MVE_0 2 "s_register_operand" "w")
4197                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4198          VCMLAQ_ROT90_F))
4199   ]
4200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4201   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4202   [(set_attr "type" "mve_move")
4203 ])
4204
4205 ;;
4206 ;; [vcmpeqq_m_n_f])
4207 ;;
4208 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4209   [
4210    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4211         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4212                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4213                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4214          VCMPEQQ_M_N_F))
4215   ]
4216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4217   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4218   [(set_attr "type" "mve_move")
4219    (set_attr "length""8")])
4220
4221 ;;
4222 ;; [vcmpgeq_m_f])
4223 ;;
4224 (define_insn "mve_vcmpgeq_m_f<mode>"
4225   [
4226    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4227         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4228                        (match_operand:MVE_0 2 "s_register_operand" "w")
4229                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4230          VCMPGEQ_M_F))
4231   ]
4232   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4233   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4234   [(set_attr "type" "mve_move")
4235    (set_attr "length""8")])
4236
4237 ;;
4238 ;; [vcmpgeq_m_n_f])
4239 ;;
4240 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4241   [
4242    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4243         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4244                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4245                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4246          VCMPGEQ_M_N_F))
4247   ]
4248   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4249   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4250   [(set_attr "type" "mve_move")
4251    (set_attr "length""8")])
4252
4253 ;;
4254 ;; [vcmpgtq_m_f])
4255 ;;
4256 (define_insn "mve_vcmpgtq_m_f<mode>"
4257   [
4258    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4259         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4260                        (match_operand:MVE_0 2 "s_register_operand" "w")
4261                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4262          VCMPGTQ_M_F))
4263   ]
4264   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4265   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4266   [(set_attr "type" "mve_move")
4267    (set_attr "length""8")])
4268
4269 ;;
4270 ;; [vcmpgtq_m_n_f])
4271 ;;
4272 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4273   [
4274    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4275         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4276                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4277                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4278          VCMPGTQ_M_N_F))
4279   ]
4280   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4281   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4282   [(set_attr "type" "mve_move")
4283    (set_attr "length""8")])
4284
4285 ;;
4286 ;; [vcmpleq_m_f])
4287 ;;
4288 (define_insn "mve_vcmpleq_m_f<mode>"
4289   [
4290    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4291         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4292                        (match_operand:MVE_0 2 "s_register_operand" "w")
4293                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4294          VCMPLEQ_M_F))
4295   ]
4296   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4297   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4298   [(set_attr "type" "mve_move")
4299    (set_attr "length""8")])
4300
4301 ;;
4302 ;; [vcmpleq_m_n_f])
4303 ;;
4304 (define_insn "mve_vcmpleq_m_n_f<mode>"
4305   [
4306    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4307         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4308                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4309                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4310          VCMPLEQ_M_N_F))
4311   ]
4312   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4313   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4314   [(set_attr "type" "mve_move")
4315    (set_attr "length""8")])
4316
4317 ;;
4318 ;; [vcmpltq_m_f])
4319 ;;
4320 (define_insn "mve_vcmpltq_m_f<mode>"
4321   [
4322    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4323         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4324                        (match_operand:MVE_0 2 "s_register_operand" "w")
4325                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4326          VCMPLTQ_M_F))
4327   ]
4328   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4329   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4330   [(set_attr "type" "mve_move")
4331    (set_attr "length""8")])
4332
4333 ;;
4334 ;; [vcmpltq_m_n_f])
4335 ;;
4336 (define_insn "mve_vcmpltq_m_n_f<mode>"
4337   [
4338    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4339         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4340                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4341                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4342          VCMPLTQ_M_N_F))
4343   ]
4344   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4345   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4346   [(set_attr "type" "mve_move")
4347    (set_attr "length""8")])
4348
4349 ;;
4350 ;; [vcmpneq_m_f])
4351 ;;
4352 (define_insn "mve_vcmpneq_m_f<mode>"
4353   [
4354    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4355         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4356                        (match_operand:MVE_0 2 "s_register_operand" "w")
4357                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4358          VCMPNEQ_M_F))
4359   ]
4360   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4361   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4362   [(set_attr "type" "mve_move")
4363    (set_attr "length""8")])
4364
4365 ;;
4366 ;; [vcmpneq_m_n_f])
4367 ;;
4368 (define_insn "mve_vcmpneq_m_n_f<mode>"
4369   [
4370    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4371         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4372                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4373                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4374          VCMPNEQ_M_N_F))
4375   ]
4376   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4377   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4378   [(set_attr "type" "mve_move")
4379    (set_attr "length""8")])
4380
4381 ;;
4382 ;; [vcvtbq_m_f16_f32])
4383 ;;
4384 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4385   [
4386    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4387         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4388                        (match_operand:V4SF 2 "s_register_operand" "w")
4389                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4390          VCVTBQ_M_F16_F32))
4391   ]
4392   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4393   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4394   [(set_attr "type" "mve_move")
4395    (set_attr "length""8")])
4396
4397 ;;
4398 ;; [vcvtbq_m_f32_f16])
4399 ;;
4400 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4401   [
4402    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4403         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4404                        (match_operand:V8HF 2 "s_register_operand" "w")
4405                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4406          VCVTBQ_M_F32_F16))
4407   ]
4408   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4409   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4410   [(set_attr "type" "mve_move")
4411    (set_attr "length""8")])
4412
4413 ;;
4414 ;; [vcvttq_m_f16_f32])
4415 ;;
4416 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4417   [
4418    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4419         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4420                        (match_operand:V4SF 2 "s_register_operand" "w")
4421                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4422          VCVTTQ_M_F16_F32))
4423   ]
4424   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4425   "vpst\;vcvttt.f16.f32 %q0, %q2"
4426   [(set_attr "type" "mve_move")
4427    (set_attr "length""8")])
4428
4429 ;;
4430 ;; [vcvttq_m_f32_f16])
4431 ;;
4432 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4433   [
4434    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4435         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4436                        (match_operand:V8HF 2 "s_register_operand" "w")
4437                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4438          VCVTTQ_M_F32_F16))
4439   ]
4440   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4441   "vpst\;vcvttt.f32.f16 %q0, %q2"
4442   [(set_attr "type" "mve_move")
4443    (set_attr "length""8")])
4444
4445 ;;
4446 ;; [vdupq_m_n_f])
4447 ;;
4448 (define_insn "mve_vdupq_m_n_f<mode>"
4449   [
4450    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4451         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4452                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4453                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4454          VDUPQ_M_N_F))
4455   ]
4456   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4457   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4458   [(set_attr "type" "mve_move")
4459    (set_attr "length""8")])
4460
4461 ;;
4462 ;; [vfmaq_f])
4463 ;;
4464 (define_insn "mve_vfmaq_f<mode>"
4465   [
4466    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4467         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4468                        (match_operand:MVE_0 2 "s_register_operand" "w")
4469                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4470          VFMAQ_F))
4471   ]
4472   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4473   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4474   [(set_attr "type" "mve_move")
4475 ])
4476
4477 ;;
4478 ;; [vfmaq_n_f])
4479 ;;
4480 (define_insn "mve_vfmaq_n_f<mode>"
4481   [
4482    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4483         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4484                        (match_operand:MVE_0 2 "s_register_operand" "w")
4485                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4486          VFMAQ_N_F))
4487   ]
4488   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4489   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4490   [(set_attr "type" "mve_move")
4491 ])
4492
4493 ;;
4494 ;; [vfmasq_n_f])
4495 ;;
4496 (define_insn "mve_vfmasq_n_f<mode>"
4497   [
4498    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4499         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4500                        (match_operand:MVE_0 2 "s_register_operand" "w")
4501                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4502          VFMASQ_N_F))
4503   ]
4504   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4505   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4506   [(set_attr "type" "mve_move")
4507 ])
4508 ;;
4509 ;; [vfmsq_f])
4510 ;;
4511 (define_insn "mve_vfmsq_f<mode>"
4512   [
4513    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4514         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4515                        (match_operand:MVE_0 2 "s_register_operand" "w")
4516                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4517          VFMSQ_F))
4518   ]
4519   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4520   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4521   [(set_attr "type" "mve_move")
4522 ])
4523
4524 ;;
4525 ;; [vmaxnmaq_m_f])
4526 ;;
4527 (define_insn "mve_vmaxnmaq_m_f<mode>"
4528   [
4529    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4530         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4531                        (match_operand:MVE_0 2 "s_register_operand" "w")
4532                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4533          VMAXNMAQ_M_F))
4534   ]
4535   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4536   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4537   [(set_attr "type" "mve_move")
4538    (set_attr "length""8")])
4539 ;;
4540 ;; [vmaxnmavq_p_f])
4541 ;;
4542 (define_insn "mve_vmaxnmavq_p_f<mode>"
4543   [
4544    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4545         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4546                        (match_operand:MVE_0 2 "s_register_operand" "w")
4547                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4548          VMAXNMAVQ_P_F))
4549   ]
4550   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4551   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4552   [(set_attr "type" "mve_move")
4553    (set_attr "length""8")])
4554
4555 ;;
4556 ;; [vmaxnmvq_p_f])
4557 ;;
4558 (define_insn "mve_vmaxnmvq_p_f<mode>"
4559   [
4560    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4561         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4562                        (match_operand:MVE_0 2 "s_register_operand" "w")
4563                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4564          VMAXNMVQ_P_F))
4565   ]
4566   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4567   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4568   [(set_attr "type" "mve_move")
4569    (set_attr "length""8")])
4570 ;;
4571 ;; [vminnmaq_m_f])
4572 ;;
4573 (define_insn "mve_vminnmaq_m_f<mode>"
4574   [
4575    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4576         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4577                        (match_operand:MVE_0 2 "s_register_operand" "w")
4578                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4579          VMINNMAQ_M_F))
4580   ]
4581   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4582   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4583   [(set_attr "type" "mve_move")
4584    (set_attr "length""8")])
4585
4586 ;;
4587 ;; [vminnmavq_p_f])
4588 ;;
4589 (define_insn "mve_vminnmavq_p_f<mode>"
4590   [
4591    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4592         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4593                        (match_operand:MVE_0 2 "s_register_operand" "w")
4594                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4595          VMINNMAVQ_P_F))
4596   ]
4597   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4598   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4599   [(set_attr "type" "mve_move")
4600    (set_attr "length""8")])
4601 ;;
4602 ;; [vminnmvq_p_f])
4603 ;;
4604 (define_insn "mve_vminnmvq_p_f<mode>"
4605   [
4606    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4607         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4608                        (match_operand:MVE_0 2 "s_register_operand" "w")
4609                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4610          VMINNMVQ_P_F))
4611   ]
4612   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4613   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4614   [(set_attr "type" "mve_move")
4615    (set_attr "length""8")])
4616
4617 ;;
4618 ;; [vmlaldavaq_s, vmlaldavaq_u])
4619 ;;
4620 (define_insn "mve_vmlaldavaq_<supf><mode>"
4621   [
4622    (set (match_operand:DI 0 "s_register_operand" "=r")
4623         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4624                        (match_operand:MVE_5 2 "s_register_operand" "w")
4625                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4626          VMLALDAVAQ))
4627   ]
4628   "TARGET_HAVE_MVE"
4629   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4630   [(set_attr "type" "mve_move")
4631 ])
4632
4633 ;;
4634 ;; [vmlaldavaxq_s])
4635 ;;
4636 (define_insn "mve_vmlaldavaxq_s<mode>"
4637   [
4638    (set (match_operand:DI 0 "s_register_operand" "=r")
4639         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4640                        (match_operand:MVE_5 2 "s_register_operand" "w")
4641                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4642          VMLALDAVAXQ_S))
4643   ]
4644   "TARGET_HAVE_MVE"
4645   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4646   [(set_attr "type" "mve_move")
4647 ])
4648
4649 ;;
4650 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4651 ;;
4652 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4653   [
4654    (set (match_operand:DI 0 "s_register_operand" "=r")
4655         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4656                        (match_operand:MVE_5 2 "s_register_operand" "w")
4657                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4658          VMLALDAVQ_P))
4659   ]
4660   "TARGET_HAVE_MVE"
4661   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4662   [(set_attr "type" "mve_move")
4663    (set_attr "length""8")])
4664
4665 ;;
4666 ;; [vmlaldavxq_p_s])
4667 ;;
4668 (define_insn "mve_vmlaldavxq_p_s<mode>"
4669   [
4670    (set (match_operand:DI 0 "s_register_operand" "=r")
4671         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4672                        (match_operand:MVE_5 2 "s_register_operand" "w")
4673                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4674          VMLALDAVXQ_P_S))
4675   ]
4676   "TARGET_HAVE_MVE"
4677   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4678   [(set_attr "type" "mve_move")
4679    (set_attr "length""8")])
4680 ;;
4681 ;; [vmlsldavaq_s])
4682 ;;
4683 (define_insn "mve_vmlsldavaq_s<mode>"
4684   [
4685    (set (match_operand:DI 0 "s_register_operand" "=r")
4686         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4687                        (match_operand:MVE_5 2 "s_register_operand" "w")
4688                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4689          VMLSLDAVAQ_S))
4690   ]
4691   "TARGET_HAVE_MVE"
4692   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4693   [(set_attr "type" "mve_move")
4694 ])
4695
4696 ;;
4697 ;; [vmlsldavaxq_s])
4698 ;;
4699 (define_insn "mve_vmlsldavaxq_s<mode>"
4700   [
4701    (set (match_operand:DI 0 "s_register_operand" "=r")
4702         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4703                        (match_operand:MVE_5 2 "s_register_operand" "w")
4704                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4705          VMLSLDAVAXQ_S))
4706   ]
4707   "TARGET_HAVE_MVE"
4708   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4709   [(set_attr "type" "mve_move")
4710 ])
4711
4712 ;;
4713 ;; [vmlsldavq_p_s])
4714 ;;
4715 (define_insn "mve_vmlsldavq_p_s<mode>"
4716   [
4717    (set (match_operand:DI 0 "s_register_operand" "=r")
4718         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4719                        (match_operand:MVE_5 2 "s_register_operand" "w")
4720                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4721          VMLSLDAVQ_P_S))
4722   ]
4723   "TARGET_HAVE_MVE"
4724   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4725   [(set_attr "type" "mve_move")
4726    (set_attr "length""8")])
4727
4728 ;;
4729 ;; [vmlsldavxq_p_s])
4730 ;;
4731 (define_insn "mve_vmlsldavxq_p_s<mode>"
4732   [
4733    (set (match_operand:DI 0 "s_register_operand" "=r")
4734         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4735                        (match_operand:MVE_5 2 "s_register_operand" "w")
4736                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4737          VMLSLDAVXQ_P_S))
4738   ]
4739   "TARGET_HAVE_MVE"
4740   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4741   [(set_attr "type" "mve_move")
4742    (set_attr "length""8")])
4743 ;;
4744 ;; [vmovlbq_m_u, vmovlbq_m_s])
4745 ;;
4746 (define_insn "mve_vmovlbq_m_<supf><mode>"
4747   [
4748    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4749         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4750                        (match_operand:MVE_3 2 "s_register_operand" "w")
4751                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4752          VMOVLBQ_M))
4753   ]
4754   "TARGET_HAVE_MVE"
4755   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4756   [(set_attr "type" "mve_move")
4757    (set_attr "length""8")])
4758 ;;
4759 ;; [vmovltq_m_u, vmovltq_m_s])
4760 ;;
4761 (define_insn "mve_vmovltq_m_<supf><mode>"
4762   [
4763    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4764         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4765                        (match_operand:MVE_3 2 "s_register_operand" "w")
4766                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4767          VMOVLTQ_M))
4768   ]
4769   "TARGET_HAVE_MVE"
4770   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4771   [(set_attr "type" "mve_move")
4772    (set_attr "length""8")])
4773 ;;
4774 ;; [vmovnbq_m_u, vmovnbq_m_s])
4775 ;;
4776 (define_insn "mve_vmovnbq_m_<supf><mode>"
4777   [
4778    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4779         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4780                        (match_operand:MVE_5 2 "s_register_operand" "w")
4781                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4782          VMOVNBQ_M))
4783   ]
4784   "TARGET_HAVE_MVE"
4785   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4786   [(set_attr "type" "mve_move")
4787    (set_attr "length""8")])
4788
4789 ;;
4790 ;; [vmovntq_m_u, vmovntq_m_s])
4791 ;;
4792 (define_insn "mve_vmovntq_m_<supf><mode>"
4793   [
4794    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4795         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4796                        (match_operand:MVE_5 2 "s_register_operand" "w")
4797                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4798          VMOVNTQ_M))
4799   ]
4800   "TARGET_HAVE_MVE"
4801   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4802   [(set_attr "type" "mve_move")
4803    (set_attr "length""8")])
4804
4805 ;;
4806 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4807 ;;
4808 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4809   [
4810    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4811         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4812                        (match_operand:SI 2 "immediate_operand" "i")
4813                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4814          VMVNQ_M_N))
4815   ]
4816   "TARGET_HAVE_MVE"
4817   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4818   [(set_attr "type" "mve_move")
4819    (set_attr "length""8")])
4820 ;;
4821 ;; [vnegq_m_f])
4822 ;;
4823 (define_insn "mve_vnegq_m_f<mode>"
4824   [
4825    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4826         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4827                        (match_operand:MVE_0 2 "s_register_operand" "w")
4828                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4829          VNEGQ_M_F))
4830   ]
4831   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4832   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4833   [(set_attr "type" "mve_move")
4834    (set_attr "length""8")])
4835
4836 ;;
4837 ;; [vorrq_m_n_s, vorrq_m_n_u])
4838 ;;
4839 (define_insn "mve_vorrq_m_n_<supf><mode>"
4840   [
4841    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4842         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4843                        (match_operand:SI 2 "immediate_operand" "i")
4844                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4845          VORRQ_M_N))
4846   ]
4847   "TARGET_HAVE_MVE"
4848   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4849   [(set_attr "type" "mve_move")
4850    (set_attr "length""8")])
4851 ;;
4852 ;; [vpselq_f])
4853 ;;
4854 (define_insn "mve_vpselq_f<mode>"
4855   [
4856    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4857         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4858                        (match_operand:MVE_0 2 "s_register_operand" "w")
4859                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4860          VPSELQ_F))
4861   ]
4862   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4863   "vpsel %q0, %q1, %q2"
4864   [(set_attr "type" "mve_move")
4865 ])
4866
4867 ;;
4868 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4869 ;;
4870 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4871   [
4872    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4873         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4874                        (match_operand:MVE_5 2 "s_register_operand" "w")
4875                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4876          VQMOVNBQ_M))
4877   ]
4878   "TARGET_HAVE_MVE"
4879   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4880   [(set_attr "type" "mve_move")
4881    (set_attr "length""8")])
4882
4883 ;;
4884 ;; [vqmovntq_m_u, vqmovntq_m_s])
4885 ;;
4886 (define_insn "mve_vqmovntq_m_<supf><mode>"
4887   [
4888    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4889         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4890                        (match_operand:MVE_5 2 "s_register_operand" "w")
4891                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4892          VQMOVNTQ_M))
4893   ]
4894   "TARGET_HAVE_MVE"
4895   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4896   [(set_attr "type" "mve_move")
4897    (set_attr "length""8")])
4898
4899 ;;
4900 ;; [vqmovunbq_m_s])
4901 ;;
4902 (define_insn "mve_vqmovunbq_m_s<mode>"
4903   [
4904    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4905         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4906                        (match_operand:MVE_5 2 "s_register_operand" "w")
4907                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4908          VQMOVUNBQ_M_S))
4909   ]
4910   "TARGET_HAVE_MVE"
4911   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4912   [(set_attr "type" "mve_move")
4913    (set_attr "length""8")])
4914
4915 ;;
4916 ;; [vqmovuntq_m_s])
4917 ;;
4918 (define_insn "mve_vqmovuntq_m_s<mode>"
4919   [
4920    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4921         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4922                        (match_operand:MVE_5 2 "s_register_operand" "w")
4923                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4924          VQMOVUNTQ_M_S))
4925   ]
4926   "TARGET_HAVE_MVE"
4927   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4928   [(set_attr "type" "mve_move")
4929    (set_attr "length""8")])
4930
4931 ;;
4932 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4933 ;;
4934 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4935   [
4936    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4937         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4938                        (match_operand:MVE_5 2 "s_register_operand" "w")
4939                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4940          VQRSHRNTQ_N))
4941   ]
4942   "TARGET_HAVE_MVE"
4943   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4944   [(set_attr "type" "mve_move")
4945 ])
4946
4947 ;;
4948 ;; [vqrshruntq_n_s])
4949 ;;
4950 (define_insn "mve_vqrshruntq_n_s<mode>"
4951   [
4952    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4953         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4954                        (match_operand:MVE_5 2 "s_register_operand" "w")
4955                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4956          VQRSHRUNTQ_N_S))
4957   ]
4958   "TARGET_HAVE_MVE"
4959   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4960   [(set_attr "type" "mve_move")
4961 ])
4962
4963 ;;
4964 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4965 ;;
4966 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4967   [
4968    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4969         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4970                        (match_operand:MVE_5 2 "s_register_operand" "w")
4971                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4972          VQSHRNBQ_N))
4973   ]
4974   "TARGET_HAVE_MVE"
4975   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4976   [(set_attr "type" "mve_move")
4977 ])
4978
4979 ;;
4980 ;; [vqshrntq_n_u, vqshrntq_n_s])
4981 ;;
4982 (define_insn "mve_vqshrntq_n_<supf><mode>"
4983   [
4984    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4985         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4986                        (match_operand:MVE_5 2 "s_register_operand" "w")
4987                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4988          VQSHRNTQ_N))
4989   ]
4990   "TARGET_HAVE_MVE"
4991   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4992   [(set_attr "type" "mve_move")
4993 ])
4994
4995 ;;
4996 ;; [vqshrunbq_n_s])
4997 ;;
4998 (define_insn "mve_vqshrunbq_n_s<mode>"
4999   [
5000    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5001         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5002                        (match_operand:MVE_5 2 "s_register_operand" "w")
5003                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5004          VQSHRUNBQ_N_S))
5005   ]
5006   "TARGET_HAVE_MVE"
5007   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5008   [(set_attr "type" "mve_move")
5009 ])
5010
5011 ;;
5012 ;; [vqshruntq_n_s])
5013 ;;
5014 (define_insn "mve_vqshruntq_n_s<mode>"
5015   [
5016    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5017         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5018                        (match_operand:MVE_5 2 "s_register_operand" "w")
5019                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5020          VQSHRUNTQ_N_S))
5021   ]
5022   "TARGET_HAVE_MVE"
5023   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5024   [(set_attr "type" "mve_move")
5025 ])
5026
5027 ;;
5028 ;; [vrev32q_m_f])
5029 ;;
5030 (define_insn "mve_vrev32q_m_fv8hf"
5031   [
5032    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5033         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5034                        (match_operand:V8HF 2 "s_register_operand" "w")
5035                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5036          VREV32Q_M_F))
5037   ]
5038   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5039   "vpst\;vrev32t.16 %q0, %q2"
5040   [(set_attr "type" "mve_move")
5041    (set_attr "length""8")])
5042
5043 ;;
5044 ;; [vrev32q_m_s, vrev32q_m_u])
5045 ;;
5046 (define_insn "mve_vrev32q_m_<supf><mode>"
5047   [
5048    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5049         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5050                        (match_operand:MVE_3 2 "s_register_operand" "w")
5051                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5052          VREV32Q_M))
5053   ]
5054   "TARGET_HAVE_MVE"
5055   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5056   [(set_attr "type" "mve_move")
5057    (set_attr "length""8")])
5058
5059 ;;
5060 ;; [vrev64q_m_f])
5061 ;;
5062 (define_insn "mve_vrev64q_m_f<mode>"
5063   [
5064    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5065         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5066                        (match_operand:MVE_0 2 "s_register_operand" "w")
5067                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5068          VREV64Q_M_F))
5069   ]
5070   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5071   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5072   [(set_attr "type" "mve_move")
5073    (set_attr "length""8")])
5074
5075 ;;
5076 ;; [vrmlaldavhaxq_s])
5077 ;;
5078 (define_insn "mve_vrmlaldavhaxq_sv4si"
5079   [
5080    (set (match_operand:DI 0 "s_register_operand" "=r")
5081         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5082                        (match_operand:V4SI 2 "s_register_operand" "w")
5083                        (match_operand:V4SI 3 "s_register_operand" "w")]
5084          VRMLALDAVHAXQ_S))
5085   ]
5086   "TARGET_HAVE_MVE"
5087   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5088   [(set_attr "type" "mve_move")
5089 ])
5090
5091 ;;
5092 ;; [vrmlaldavhxq_p_s])
5093 ;;
5094 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5095   [
5096    (set (match_operand:DI 0 "s_register_operand" "=r")
5097         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5098                        (match_operand:V4SI 2 "s_register_operand" "w")
5099                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5100          VRMLALDAVHXQ_P_S))
5101   ]
5102   "TARGET_HAVE_MVE"
5103   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5104   [(set_attr "type" "mve_move")
5105    (set_attr "length""8")])
5106
5107 ;;
5108 ;; [vrmlsldavhaxq_s])
5109 ;;
5110 (define_insn "mve_vrmlsldavhaxq_sv4si"
5111   [
5112    (set (match_operand:DI 0 "s_register_operand" "=r")
5113         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5114                        (match_operand:V4SI 2 "s_register_operand" "w")
5115                        (match_operand:V4SI 3 "s_register_operand" "w")]
5116          VRMLSLDAVHAXQ_S))
5117   ]
5118   "TARGET_HAVE_MVE"
5119   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5120   [(set_attr "type" "mve_move")
5121 ])
5122
5123 ;;
5124 ;; [vrmlsldavhq_p_s])
5125 ;;
5126 (define_insn "mve_vrmlsldavhq_p_sv4si"
5127   [
5128    (set (match_operand:DI 0 "s_register_operand" "=r")
5129         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5130                        (match_operand:V4SI 2 "s_register_operand" "w")
5131                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5132          VRMLSLDAVHQ_P_S))
5133   ]
5134   "TARGET_HAVE_MVE"
5135   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5136   [(set_attr "type" "mve_move")
5137    (set_attr "length""8")])
5138
5139 ;;
5140 ;; [vrmlsldavhxq_p_s])
5141 ;;
5142 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5143   [
5144    (set (match_operand:DI 0 "s_register_operand" "=r")
5145         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5146                        (match_operand:V4SI 2 "s_register_operand" "w")
5147                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5148          VRMLSLDAVHXQ_P_S))
5149   ]
5150   "TARGET_HAVE_MVE"
5151   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5152   [(set_attr "type" "mve_move")
5153    (set_attr "length""8")])
5154
5155 ;;
5156 ;; [vrndaq_m_f])
5157 ;;
5158 (define_insn "mve_vrndaq_m_f<mode>"
5159   [
5160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5162                        (match_operand:MVE_0 2 "s_register_operand" "w")
5163                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5164          VRNDAQ_M_F))
5165   ]
5166   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5167   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5168   [(set_attr "type" "mve_move")
5169    (set_attr "length""8")])
5170
5171 ;;
5172 ;; [vrndmq_m_f])
5173 ;;
5174 (define_insn "mve_vrndmq_m_f<mode>"
5175   [
5176    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5177         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5178                        (match_operand:MVE_0 2 "s_register_operand" "w")
5179                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5180          VRNDMQ_M_F))
5181   ]
5182   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5183   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5184   [(set_attr "type" "mve_move")
5185    (set_attr "length""8")])
5186
5187 ;;
5188 ;; [vrndnq_m_f])
5189 ;;
5190 (define_insn "mve_vrndnq_m_f<mode>"
5191   [
5192    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5193         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5194                        (match_operand:MVE_0 2 "s_register_operand" "w")
5195                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5196          VRNDNQ_M_F))
5197   ]
5198   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5199   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5200   [(set_attr "type" "mve_move")
5201    (set_attr "length""8")])
5202
5203 ;;
5204 ;; [vrndpq_m_f])
5205 ;;
5206 (define_insn "mve_vrndpq_m_f<mode>"
5207   [
5208    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5209         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5210                        (match_operand:MVE_0 2 "s_register_operand" "w")
5211                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5212          VRNDPQ_M_F))
5213   ]
5214   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5215   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5216   [(set_attr "type" "mve_move")
5217    (set_attr "length""8")])
5218
5219 ;;
5220 ;; [vrndxq_m_f])
5221 ;;
5222 (define_insn "mve_vrndxq_m_f<mode>"
5223   [
5224    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5225         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5226                        (match_operand:MVE_0 2 "s_register_operand" "w")
5227                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5228          VRNDXQ_M_F))
5229   ]
5230   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5231   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5232   [(set_attr "type" "mve_move")
5233    (set_attr "length""8")])
5234
5235 ;;
5236 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5237 ;;
5238 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5239   [
5240    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5241         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5242                        (match_operand:MVE_5 2 "s_register_operand" "w")
5243                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5244          VRSHRNBQ_N))
5245   ]
5246   "TARGET_HAVE_MVE"
5247   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5248   [(set_attr "type" "mve_move")
5249 ])
5250
5251 ;;
5252 ;; [vrshrntq_n_u, vrshrntq_n_s])
5253 ;;
5254 (define_insn "mve_vrshrntq_n_<supf><mode>"
5255   [
5256    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5257         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5258                        (match_operand:MVE_5 2 "s_register_operand" "w")
5259                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5260          VRSHRNTQ_N))
5261   ]
5262   "TARGET_HAVE_MVE"
5263   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5264   [(set_attr "type" "mve_move")
5265 ])
5266
5267 ;;
5268 ;; [vshrnbq_n_u, vshrnbq_n_s])
5269 ;;
5270 (define_insn "mve_vshrnbq_n_<supf><mode>"
5271   [
5272    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5273         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5274                        (match_operand:MVE_5 2 "s_register_operand" "w")
5275                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5276          VSHRNBQ_N))
5277   ]
5278   "TARGET_HAVE_MVE"
5279   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5280   [(set_attr "type" "mve_move")
5281 ])
5282
5283 ;;
5284 ;; [vshrntq_n_s, vshrntq_n_u])
5285 ;;
5286 (define_insn "mve_vshrntq_n_<supf><mode>"
5287   [
5288    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5289         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5290                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5291                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5292          VSHRNTQ_N))
5293   ]
5294   "TARGET_HAVE_MVE"
5295   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5296   [(set_attr "type" "mve_move")
5297 ])
5298
5299 ;;
5300 ;; [vcvtmq_m_s, vcvtmq_m_u])
5301 ;;
5302 (define_insn "mve_vcvtmq_m_<supf><mode>"
5303   [
5304    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5305         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5306                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5307                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5308          VCVTMQ_M))
5309   ]
5310   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5311   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5312   [(set_attr "type" "mve_move")
5313    (set_attr "length""8")])
5314
5315 ;;
5316 ;; [vcvtpq_m_u, vcvtpq_m_s])
5317 ;;
5318 (define_insn "mve_vcvtpq_m_<supf><mode>"
5319   [
5320    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5321         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5322                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5323                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5324          VCVTPQ_M))
5325   ]
5326   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5327   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5328   [(set_attr "type" "mve_move")
5329    (set_attr "length""8")])
5330
5331 ;;
5332 ;; [vcvtnq_m_s, vcvtnq_m_u])
5333 ;;
5334 (define_insn "mve_vcvtnq_m_<supf><mode>"
5335   [
5336    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5337         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5338                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5339                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5340          VCVTNQ_M))
5341   ]
5342   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5343   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5344   [(set_attr "type" "mve_move")
5345    (set_attr "length""8")])
5346
5347 ;;
5348 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5349 ;;
5350 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5351   [
5352    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5353         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5354                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5355                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5356                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5357          VCVTQ_M_N_FROM_F))
5358   ]
5359   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5360   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5361   [(set_attr "type" "mve_move")
5362    (set_attr "length""8")])
5363
5364 ;;
5365 ;; [vrev16q_m_u, vrev16q_m_s])
5366 ;;
5367 (define_insn "mve_vrev16q_m_<supf>v16qi"
5368   [
5369    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5370         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5371                        (match_operand:V16QI 2 "s_register_operand" "w")
5372                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5373          VREV16Q_M))
5374   ]
5375   "TARGET_HAVE_MVE"
5376   "vpst\;vrev16t.8 %q0, %q2"
5377   [(set_attr "type" "mve_move")
5378    (set_attr "length""8")])
5379
5380 ;;
5381 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5382 ;;
5383 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5384   [
5385    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5386         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5387                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5388                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5389          VCVTQ_M_FROM_F))
5390   ]
5391   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5392   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5393   [(set_attr "type" "mve_move")
5394    (set_attr "length""8")])
5395
5396 ;;
5397 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5398 ;;
5399 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5400   [
5401    (set (match_operand:DI 0 "s_register_operand" "=r")
5402         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5403                     (match_operand:V4SI 2 "s_register_operand" "w")
5404                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5405          VRMLALDAVHQ_P))
5406   ]
5407   "TARGET_HAVE_MVE"
5408   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5409   [(set_attr "type" "mve_move")
5410    (set_attr "length""8")])
5411
5412 ;;
5413 ;; [vrmlsldavhaq_s])
5414 ;;
5415 (define_insn "mve_vrmlsldavhaq_sv4si"
5416   [
5417    (set (match_operand:DI 0 "s_register_operand" "=r")
5418         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5419                     (match_operand:V4SI 2 "s_register_operand" "w")
5420                     (match_operand:V4SI 3 "s_register_operand" "w")]
5421          VRMLSLDAVHAQ_S))
5422   ]
5423   "TARGET_HAVE_MVE"
5424   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5425   [(set_attr "type" "mve_move")
5426 ])
5427
5428 ;;
5429 ;; [vabavq_p_s, vabavq_p_u])
5430 ;;
5431 (define_insn "mve_vabavq_p_<supf><mode>"
5432   [
5433    (set (match_operand:SI 0 "s_register_operand" "=r")
5434         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5435                     (match_operand:MVE_2 2 "s_register_operand" "w")
5436                     (match_operand:MVE_2 3 "s_register_operand" "w")
5437                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5438          VABAVQ_P))
5439   ]
5440   "TARGET_HAVE_MVE"
5441   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5442   [(set_attr "type" "mve_move")
5443 ])
5444
5445 ;;
5446 ;; [vqshluq_m_n_s])
5447 ;;
5448 (define_insn "mve_vqshluq_m_n_s<mode>"
5449   [
5450    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5451         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5452                        (match_operand:MVE_2 2 "s_register_operand" "w")
5453                        (match_operand:SI 3 "mve_imm_7" "Ra")
5454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5455          VQSHLUQ_M_N_S))
5456   ]
5457   "TARGET_HAVE_MVE"
5458   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5459   [(set_attr "type" "mve_move")])
5460
5461 ;;
5462 ;; [vshlq_m_s, vshlq_m_u])
5463 ;;
5464 (define_insn "mve_vshlq_m_<supf><mode>"
5465   [
5466    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5467         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5468                        (match_operand:MVE_2 2 "s_register_operand" "w")
5469                        (match_operand:MVE_2 3 "s_register_operand" "w")
5470                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5471          VSHLQ_M))
5472   ]
5473   "TARGET_HAVE_MVE"
5474   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5475   [(set_attr "type" "mve_move")])
5476
5477 ;;
5478 ;; [vsriq_m_n_s, vsriq_m_n_u])
5479 ;;
5480 (define_insn "mve_vsriq_m_n_<supf><mode>"
5481   [
5482    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5483         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5484                        (match_operand:MVE_2 2 "s_register_operand" "w")
5485                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5486                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5487          VSRIQ_M_N))
5488   ]
5489   "TARGET_HAVE_MVE"
5490   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5491   [(set_attr "type" "mve_move")])
5492
5493 ;;
5494 ;; [vsubq_m_u, vsubq_m_s])
5495 ;;
5496 (define_insn "mve_vsubq_m_<supf><mode>"
5497   [
5498    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5499         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5500                        (match_operand:MVE_2 2 "s_register_operand" "w")
5501                        (match_operand:MVE_2 3 "s_register_operand" "w")
5502                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5503          VSUBQ_M))
5504   ]
5505   "TARGET_HAVE_MVE"
5506   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5507   [(set_attr "type" "mve_move")])
5508
5509 ;;
5510 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5511 ;;
5512 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5513   [
5514    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5515         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5516                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5517                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5518                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5519          VCVTQ_M_N_TO_F))
5520   ]
5521   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5522   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5523   [(set_attr "type" "mve_move")
5524    (set_attr "length""8")])
5525 ;;
5526 ;; [vabdq_m_s, vabdq_m_u])
5527 ;;
5528 (define_insn "mve_vabdq_m_<supf><mode>"
5529   [
5530    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5531         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5532                        (match_operand:MVE_2 2 "s_register_operand" "w")
5533                        (match_operand:MVE_2 3 "s_register_operand" "w")
5534                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5535          VABDQ_M))
5536   ]
5537   "TARGET_HAVE_MVE"
5538   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5539   [(set_attr "type" "mve_move")
5540    (set_attr "length""8")])
5541
5542 ;;
5543 ;; [vaddq_m_n_s, vaddq_m_n_u])
5544 ;;
5545 (define_insn "mve_vaddq_m_n_<supf><mode>"
5546   [
5547    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5548         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5549                        (match_operand:MVE_2 2 "s_register_operand" "w")
5550                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5551                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5552          VADDQ_M_N))
5553   ]
5554   "TARGET_HAVE_MVE"
5555   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5556   [(set_attr "type" "mve_move")
5557    (set_attr "length""8")])
5558
5559 ;;
5560 ;; [vaddq_m_u, vaddq_m_s])
5561 ;;
5562 (define_insn "mve_vaddq_m_<supf><mode>"
5563   [
5564    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5565         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5566                        (match_operand:MVE_2 2 "s_register_operand" "w")
5567                        (match_operand:MVE_2 3 "s_register_operand" "w")
5568                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5569          VADDQ_M))
5570   ]
5571   "TARGET_HAVE_MVE"
5572   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5573   [(set_attr "type" "mve_move")
5574    (set_attr "length""8")])
5575
5576 ;;
5577 ;; [vandq_m_u, vandq_m_s])
5578 ;;
5579 (define_insn "mve_vandq_m_<supf><mode>"
5580   [
5581    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5582         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5583                        (match_operand:MVE_2 2 "s_register_operand" "w")
5584                        (match_operand:MVE_2 3 "s_register_operand" "w")
5585                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5586          VANDQ_M))
5587   ]
5588   "TARGET_HAVE_MVE"
5589   "vpst\;vandt %q0, %q2, %q3"
5590   [(set_attr "type" "mve_move")
5591    (set_attr "length""8")])
5592
5593 ;;
5594 ;; [vbicq_m_u, vbicq_m_s])
5595 ;;
5596 (define_insn "mve_vbicq_m_<supf><mode>"
5597   [
5598    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5599         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5600                        (match_operand:MVE_2 2 "s_register_operand" "w")
5601                        (match_operand:MVE_2 3 "s_register_operand" "w")
5602                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5603          VBICQ_M))
5604   ]
5605   "TARGET_HAVE_MVE"
5606   "vpst\;vbict %q0, %q2, %q3"
5607   [(set_attr "type" "mve_move")
5608    (set_attr "length""8")])
5609
5610 ;;
5611 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5612 ;;
5613 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5614   [
5615    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5616         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5617                        (match_operand:MVE_2 2 "s_register_operand" "w")
5618                        (match_operand:SI 3 "s_register_operand" "r")
5619                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5620          VBRSRQ_M_N))
5621   ]
5622   "TARGET_HAVE_MVE"
5623   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5624   [(set_attr "type" "mve_move")
5625    (set_attr "length""8")])
5626
5627 ;;
5628 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5629 ;;
5630 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5631   [
5632    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5633         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5634                        (match_operand:MVE_2 2 "s_register_operand" "w")
5635                        (match_operand:MVE_2 3 "s_register_operand" "w")
5636                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5637          VCADDQ_ROT270_M))
5638   ]
5639   "TARGET_HAVE_MVE"
5640   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5641   [(set_attr "type" "mve_move")
5642    (set_attr "length""8")])
5643
5644 ;;
5645 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5646 ;;
5647 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5648   [
5649    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5650         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5651                        (match_operand:MVE_2 2 "s_register_operand" "w")
5652                        (match_operand:MVE_2 3 "s_register_operand" "w")
5653                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5654          VCADDQ_ROT90_M))
5655   ]
5656   "TARGET_HAVE_MVE"
5657   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5658   [(set_attr "type" "mve_move")
5659    (set_attr "length""8")])
5660
5661 ;;
5662 ;; [veorq_m_s, veorq_m_u])
5663 ;;
5664 (define_insn "mve_veorq_m_<supf><mode>"
5665   [
5666    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5667         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5668                        (match_operand:MVE_2 2 "s_register_operand" "w")
5669                        (match_operand:MVE_2 3 "s_register_operand" "w")
5670                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5671          VEORQ_M))
5672   ]
5673   "TARGET_HAVE_MVE"
5674   "vpst\;veort %q0, %q2, %q3"
5675   [(set_attr "type" "mve_move")
5676    (set_attr "length""8")])
5677
5678 ;;
5679 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5680 ;;
5681 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5682   [
5683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5685                        (match_operand:MVE_2 2 "s_register_operand" "w")
5686                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5687                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5688          VHADDQ_M_N))
5689   ]
5690   "TARGET_HAVE_MVE"
5691   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5692   [(set_attr "type" "mve_move")
5693    (set_attr "length""8")])
5694
5695 ;;
5696 ;; [vhaddq_m_s, vhaddq_m_u])
5697 ;;
5698 (define_insn "mve_vhaddq_m_<supf><mode>"
5699   [
5700    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5701         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5702                        (match_operand:MVE_2 2 "s_register_operand" "w")
5703                        (match_operand:MVE_2 3 "s_register_operand" "w")
5704                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5705          VHADDQ_M))
5706   ]
5707   "TARGET_HAVE_MVE"
5708   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5709   [(set_attr "type" "mve_move")
5710    (set_attr "length""8")])
5711
5712 ;;
5713 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5714 ;;
5715 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5716   [
5717    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5718         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5719                        (match_operand:MVE_2 2 "s_register_operand" "w")
5720                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5721                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5722          VHSUBQ_M_N))
5723   ]
5724   "TARGET_HAVE_MVE"
5725   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5726   [(set_attr "type" "mve_move")
5727    (set_attr "length""8")])
5728
5729 ;;
5730 ;; [vhsubq_m_s, vhsubq_m_u])
5731 ;;
5732 (define_insn "mve_vhsubq_m_<supf><mode>"
5733   [
5734    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5735         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5736                        (match_operand:MVE_2 2 "s_register_operand" "w")
5737                        (match_operand:MVE_2 3 "s_register_operand" "w")
5738                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5739          VHSUBQ_M))
5740   ]
5741   "TARGET_HAVE_MVE"
5742   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5743   [(set_attr "type" "mve_move")
5744    (set_attr "length""8")])
5745
5746 ;;
5747 ;; [vmaxq_m_s, vmaxq_m_u])
5748 ;;
5749 (define_insn "mve_vmaxq_m_<supf><mode>"
5750   [
5751    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5752         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5753                        (match_operand:MVE_2 2 "s_register_operand" "w")
5754                        (match_operand:MVE_2 3 "s_register_operand" "w")
5755                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5756          VMAXQ_M))
5757   ]
5758   "TARGET_HAVE_MVE"
5759   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5760   [(set_attr "type" "mve_move")
5761    (set_attr "length""8")])
5762
5763 ;;
5764 ;; [vminq_m_s, vminq_m_u])
5765 ;;
5766 (define_insn "mve_vminq_m_<supf><mode>"
5767   [
5768    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5769         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5770                        (match_operand:MVE_2 2 "s_register_operand" "w")
5771                        (match_operand:MVE_2 3 "s_register_operand" "w")
5772                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5773          VMINQ_M))
5774   ]
5775   "TARGET_HAVE_MVE"
5776   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5777   [(set_attr "type" "mve_move")
5778    (set_attr "length""8")])
5779
5780 ;;
5781 ;; [vmladavaq_p_u, vmladavaq_p_s])
5782 ;;
5783 (define_insn "mve_vmladavaq_p_<supf><mode>"
5784   [
5785    (set (match_operand:SI 0 "s_register_operand" "=Te")
5786         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5787                     (match_operand:MVE_2 2 "s_register_operand" "w")
5788                     (match_operand:MVE_2 3 "s_register_operand" "w")
5789                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5790          VMLADAVAQ_P))
5791   ]
5792   "TARGET_HAVE_MVE"
5793   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5794   [(set_attr "type" "mve_move")
5795    (set_attr "length""8")])
5796
5797 ;;
5798 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5799 ;;
5800 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5801   [
5802    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5803         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5804                        (match_operand:MVE_2 2 "s_register_operand" "w")
5805                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5806                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5807          VMLAQ_M_N))
5808   ]
5809   "TARGET_HAVE_MVE"
5810   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5811   [(set_attr "type" "mve_move")
5812    (set_attr "length""8")])
5813
5814 ;;
5815 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5816 ;;
5817 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5818   [
5819    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5820         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5821                        (match_operand:MVE_2 2 "s_register_operand" "w")
5822                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5823                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5824          VMLASQ_M_N))
5825   ]
5826   "TARGET_HAVE_MVE"
5827   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5828   [(set_attr "type" "mve_move")
5829    (set_attr "length""8")])
5830
5831 ;;
5832 ;; [vmulhq_m_s, vmulhq_m_u])
5833 ;;
5834 (define_insn "mve_vmulhq_m_<supf><mode>"
5835   [
5836    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5837         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5838                        (match_operand:MVE_2 2 "s_register_operand" "w")
5839                        (match_operand:MVE_2 3 "s_register_operand" "w")
5840                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5841          VMULHQ_M))
5842   ]
5843   "TARGET_HAVE_MVE"
5844   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5845   [(set_attr "type" "mve_move")
5846    (set_attr "length""8")])
5847
5848 ;;
5849 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5850 ;;
5851 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5852   [
5853    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5854         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5855                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5856                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5857                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5858          VMULLBQ_INT_M))
5859   ]
5860   "TARGET_HAVE_MVE"
5861   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5862   [(set_attr "type" "mve_move")
5863    (set_attr "length""8")])
5864
5865 ;;
5866 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5867 ;;
5868 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5869   [
5870    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5871         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5872                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5873                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5874                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5875          VMULLTQ_INT_M))
5876   ]
5877   "TARGET_HAVE_MVE"
5878   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5879   [(set_attr "type" "mve_move")
5880    (set_attr "length""8")])
5881
5882 ;;
5883 ;; [vmulq_m_n_u, vmulq_m_n_s])
5884 ;;
5885 (define_insn "mve_vmulq_m_n_<supf><mode>"
5886   [
5887    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5888         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5889                        (match_operand:MVE_2 2 "s_register_operand" "w")
5890                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5891                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5892          VMULQ_M_N))
5893   ]
5894   "TARGET_HAVE_MVE"
5895   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5896   [(set_attr "type" "mve_move")
5897    (set_attr "length""8")])
5898
5899 ;;
5900 ;; [vmulq_m_s, vmulq_m_u])
5901 ;;
5902 (define_insn "mve_vmulq_m_<supf><mode>"
5903   [
5904    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5905         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5906                        (match_operand:MVE_2 2 "s_register_operand" "w")
5907                        (match_operand:MVE_2 3 "s_register_operand" "w")
5908                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5909          VMULQ_M))
5910   ]
5911   "TARGET_HAVE_MVE"
5912   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5913   [(set_attr "type" "mve_move")
5914    (set_attr "length""8")])
5915
5916 ;;
5917 ;; [vornq_m_u, vornq_m_s])
5918 ;;
5919 (define_insn "mve_vornq_m_<supf><mode>"
5920   [
5921    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5922         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5923                        (match_operand:MVE_2 2 "s_register_operand" "w")
5924                        (match_operand:MVE_2 3 "s_register_operand" "w")
5925                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5926          VORNQ_M))
5927   ]
5928   "TARGET_HAVE_MVE"
5929   "vpst\;vornt %q0, %q2, %q3"
5930   [(set_attr "type" "mve_move")
5931    (set_attr "length""8")])
5932
5933 ;;
5934 ;; [vorrq_m_s, vorrq_m_u])
5935 ;;
5936 (define_insn "mve_vorrq_m_<supf><mode>"
5937   [
5938    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5939         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5940                        (match_operand:MVE_2 2 "s_register_operand" "w")
5941                        (match_operand:MVE_2 3 "s_register_operand" "w")
5942                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5943          VORRQ_M))
5944   ]
5945   "TARGET_HAVE_MVE"
5946   "vpst\;vorrt %q0, %q2, %q3"
5947   [(set_attr "type" "mve_move")
5948    (set_attr "length""8")])
5949
5950 ;;
5951 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5952 ;;
5953 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5954   [
5955    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5956         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5957                        (match_operand:MVE_2 2 "s_register_operand" "w")
5958                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5959                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5960          VQADDQ_M_N))
5961   ]
5962   "TARGET_HAVE_MVE"
5963   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5964   [(set_attr "type" "mve_move")
5965    (set_attr "length""8")])
5966
5967 ;;
5968 ;; [vqaddq_m_u, vqaddq_m_s])
5969 ;;
5970 (define_insn "mve_vqaddq_m_<supf><mode>"
5971   [
5972    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5973         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5974                        (match_operand:MVE_2 2 "s_register_operand" "w")
5975                        (match_operand:MVE_2 3 "s_register_operand" "w")
5976                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5977          VQADDQ_M))
5978   ]
5979   "TARGET_HAVE_MVE"
5980   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5981   [(set_attr "type" "mve_move")
5982    (set_attr "length""8")])
5983
5984 ;;
5985 ;; [vqdmlahq_m_n_s])
5986 ;;
5987 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5988   [
5989    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5990         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5991                        (match_operand:MVE_2 2 "s_register_operand" "w")
5992                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5993                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5994          VQDMLAHQ_M_N_S))
5995   ]
5996   "TARGET_HAVE_MVE"
5997   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5998   [(set_attr "type" "mve_move")
5999    (set_attr "length""8")])
6000
6001 ;;
6002 ;; [vqdmlashq_m_n_s])
6003 ;;
6004 (define_insn "mve_vqdmlashq_m_n_s<mode>"
6005   [
6006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6008                        (match_operand:MVE_2 2 "s_register_operand" "w")
6009                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6010                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6011          VQDMLASHQ_M_N_S))
6012   ]
6013   "TARGET_HAVE_MVE"
6014   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6015   [(set_attr "type" "mve_move")
6016    (set_attr "length""8")])
6017
6018 ;;
6019 ;; [vqrdmlahq_m_n_s])
6020 ;;
6021 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6022   [
6023    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6024         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6025                        (match_operand:MVE_2 2 "s_register_operand" "w")
6026                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6027                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6028          VQRDMLAHQ_M_N_S))
6029   ]
6030   "TARGET_HAVE_MVE"
6031   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6032   [(set_attr "type" "mve_move")
6033    (set_attr "length""8")])
6034
6035 ;;
6036 ;; [vqrdmlashq_m_n_s])
6037 ;;
6038 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6039   [
6040    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6041         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6042                        (match_operand:MVE_2 2 "s_register_operand" "w")
6043                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6044                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6045          VQRDMLASHQ_M_N_S))
6046   ]
6047   "TARGET_HAVE_MVE"
6048   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6049   [(set_attr "type" "mve_move")
6050    (set_attr "length""8")])
6051
6052 ;;
6053 ;; [vqrshlq_m_u, vqrshlq_m_s])
6054 ;;
6055 (define_insn "mve_vqrshlq_m_<supf><mode>"
6056   [
6057    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6058         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6059                        (match_operand:MVE_2 2 "s_register_operand" "w")
6060                        (match_operand:MVE_2 3 "s_register_operand" "w")
6061                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6062          VQRSHLQ_M))
6063   ]
6064   "TARGET_HAVE_MVE"
6065   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6066   [(set_attr "type" "mve_move")
6067    (set_attr "length""8")])
6068
6069 ;;
6070 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6071 ;;
6072 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6073   [
6074    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6075         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6076                        (match_operand:MVE_2 2 "s_register_operand" "w")
6077                        (match_operand:SI 3 "immediate_operand" "i")
6078                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6079          VQSHLQ_M_N))
6080   ]
6081   "TARGET_HAVE_MVE"
6082   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6083   [(set_attr "type" "mve_move")
6084    (set_attr "length""8")])
6085
6086 ;;
6087 ;; [vqshlq_m_u, vqshlq_m_s])
6088 ;;
6089 (define_insn "mve_vqshlq_m_<supf><mode>"
6090   [
6091    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6092         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6093                        (match_operand:MVE_2 2 "s_register_operand" "w")
6094                        (match_operand:MVE_2 3 "s_register_operand" "w")
6095                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6096          VQSHLQ_M))
6097   ]
6098   "TARGET_HAVE_MVE"
6099   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6100   [(set_attr "type" "mve_move")
6101    (set_attr "length""8")])
6102
6103 ;;
6104 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6105 ;;
6106 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6107   [
6108    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6109         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6110                        (match_operand:MVE_2 2 "s_register_operand" "w")
6111                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6112                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6113          VQSUBQ_M_N))
6114   ]
6115   "TARGET_HAVE_MVE"
6116   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6117   [(set_attr "type" "mve_move")
6118    (set_attr "length""8")])
6119
6120 ;;
6121 ;; [vqsubq_m_u, vqsubq_m_s])
6122 ;;
6123 (define_insn "mve_vqsubq_m_<supf><mode>"
6124   [
6125    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6126         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6127                        (match_operand:MVE_2 2 "s_register_operand" "w")
6128                        (match_operand:MVE_2 3 "s_register_operand" "w")
6129                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6130          VQSUBQ_M))
6131   ]
6132   "TARGET_HAVE_MVE"
6133   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6134   [(set_attr "type" "mve_move")
6135    (set_attr "length""8")])
6136
6137 ;;
6138 ;; [vrhaddq_m_u, vrhaddq_m_s])
6139 ;;
6140 (define_insn "mve_vrhaddq_m_<supf><mode>"
6141   [
6142    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6143         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6144                        (match_operand:MVE_2 2 "s_register_operand" "w")
6145                        (match_operand:MVE_2 3 "s_register_operand" "w")
6146                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6147          VRHADDQ_M))
6148   ]
6149   "TARGET_HAVE_MVE"
6150   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6151   [(set_attr "type" "mve_move")
6152    (set_attr "length""8")])
6153
6154 ;;
6155 ;; [vrmulhq_m_u, vrmulhq_m_s])
6156 ;;
6157 (define_insn "mve_vrmulhq_m_<supf><mode>"
6158   [
6159    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6160         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6161                        (match_operand:MVE_2 2 "s_register_operand" "w")
6162                        (match_operand:MVE_2 3 "s_register_operand" "w")
6163                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6164          VRMULHQ_M))
6165   ]
6166   "TARGET_HAVE_MVE"
6167   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6168   [(set_attr "type" "mve_move")
6169    (set_attr "length""8")])
6170
6171 ;;
6172 ;; [vrshlq_m_s, vrshlq_m_u])
6173 ;;
6174 (define_insn "mve_vrshlq_m_<supf><mode>"
6175   [
6176    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6177         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6178                        (match_operand:MVE_2 2 "s_register_operand" "w")
6179                        (match_operand:MVE_2 3 "s_register_operand" "w")
6180                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6181          VRSHLQ_M))
6182   ]
6183   "TARGET_HAVE_MVE"
6184   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6185   [(set_attr "type" "mve_move")
6186    (set_attr "length""8")])
6187
6188 ;;
6189 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6190 ;;
6191 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6192   [
6193    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6194         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6195                        (match_operand:MVE_2 2 "s_register_operand" "w")
6196                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6197                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6198          VRSHRQ_M_N))
6199   ]
6200   "TARGET_HAVE_MVE"
6201   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6202   [(set_attr "type" "mve_move")
6203    (set_attr "length""8")])
6204
6205 ;;
6206 ;; [vshlq_m_n_s, vshlq_m_n_u])
6207 ;;
6208 (define_insn "mve_vshlq_m_n_<supf><mode>"
6209   [
6210    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6211         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6212                        (match_operand:MVE_2 2 "s_register_operand" "w")
6213                        (match_operand:SI 3 "immediate_operand" "i")
6214                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6215          VSHLQ_M_N))
6216   ]
6217   "TARGET_HAVE_MVE"
6218   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6219   [(set_attr "type" "mve_move")
6220    (set_attr "length""8")])
6221
6222 ;;
6223 ;; [vshrq_m_n_s, vshrq_m_n_u])
6224 ;;
6225 (define_insn "mve_vshrq_m_n_<supf><mode>"
6226   [
6227    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6228         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6229                        (match_operand:MVE_2 2 "s_register_operand" "w")
6230                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6231                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6232          VSHRQ_M_N))
6233   ]
6234   "TARGET_HAVE_MVE"
6235   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6236   [(set_attr "type" "mve_move")
6237    (set_attr "length""8")])
6238
6239 ;;
6240 ;; [vsliq_m_n_u, vsliq_m_n_s])
6241 ;;
6242 (define_insn "mve_vsliq_m_n_<supf><mode>"
6243    [
6244    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6245        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6246                        (match_operand:MVE_2 2 "s_register_operand" "w")
6247                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6248                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6249          VSLIQ_M_N))
6250   ]
6251   "TARGET_HAVE_MVE"
6252   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6253   [(set_attr "type" "mve_move")
6254    (set_attr "length""8")])
6255
6256 ;;
6257 ;; [vsubq_m_n_s, vsubq_m_n_u])
6258 ;;
6259 (define_insn "mve_vsubq_m_n_<supf><mode>"
6260   [
6261    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6262         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6263                        (match_operand:MVE_2 2 "s_register_operand" "w")
6264                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6265                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6266          VSUBQ_M_N))
6267   ]
6268   "TARGET_HAVE_MVE"
6269   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6270   [(set_attr "type" "mve_move")
6271    (set_attr "length""8")])
6272
6273 ;;
6274 ;; [vhcaddq_rot270_m_s])
6275 ;;
6276 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6277   [
6278    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6279         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6280                        (match_operand:MVE_2 2 "s_register_operand" "w")
6281                        (match_operand:MVE_2 3 "s_register_operand" "w")
6282                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6283          VHCADDQ_ROT270_M_S))
6284   ]
6285   "TARGET_HAVE_MVE"
6286   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6287   [(set_attr "type" "mve_move")
6288    (set_attr "length""8")])
6289
6290 ;;
6291 ;; [vhcaddq_rot90_m_s])
6292 ;;
6293 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6294   [
6295    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6296         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6297                        (match_operand:MVE_2 2 "s_register_operand" "w")
6298                        (match_operand:MVE_2 3 "s_register_operand" "w")
6299                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6300          VHCADDQ_ROT90_M_S))
6301   ]
6302   "TARGET_HAVE_MVE"
6303   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6304   [(set_attr "type" "mve_move")
6305    (set_attr "length""8")])
6306
6307 ;;
6308 ;; [vmladavaxq_p_s])
6309 ;;
6310 (define_insn "mve_vmladavaxq_p_s<mode>"
6311   [
6312    (set (match_operand:SI 0 "s_register_operand" "=Te")
6313         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6314                        (match_operand:MVE_2 2 "s_register_operand" "w")
6315                        (match_operand:MVE_2 3 "s_register_operand" "w")
6316                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6317          VMLADAVAXQ_P_S))
6318   ]
6319   "TARGET_HAVE_MVE"
6320   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6321   [(set_attr "type" "mve_move")
6322    (set_attr "length""8")])
6323
6324 ;;
6325 ;; [vmlsdavaq_p_s])
6326 ;;
6327 (define_insn "mve_vmlsdavaq_p_s<mode>"
6328   [
6329    (set (match_operand:SI 0 "s_register_operand" "=Te")
6330         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6331                        (match_operand:MVE_2 2 "s_register_operand" "w")
6332                        (match_operand:MVE_2 3 "s_register_operand" "w")
6333                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6334          VMLSDAVAQ_P_S))
6335   ]
6336   "TARGET_HAVE_MVE"
6337   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6338   [(set_attr "type" "mve_move")
6339    (set_attr "length""8")])
6340
6341 ;;
6342 ;; [vmlsdavaxq_p_s])
6343 ;;
6344 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6345   [
6346    (set (match_operand:SI 0 "s_register_operand" "=Te")
6347         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6348                        (match_operand:MVE_2 2 "s_register_operand" "w")
6349                        (match_operand:MVE_2 3 "s_register_operand" "w")
6350                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6351          VMLSDAVAXQ_P_S))
6352   ]
6353   "TARGET_HAVE_MVE"
6354   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6355   [(set_attr "type" "mve_move")
6356    (set_attr "length""8")])
6357
6358 ;;
6359 ;; [vqdmladhq_m_s])
6360 ;;
6361 (define_insn "mve_vqdmladhq_m_s<mode>"
6362   [
6363    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6364         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6365                        (match_operand:MVE_2 2 "s_register_operand" "w")
6366                        (match_operand:MVE_2 3 "s_register_operand" "w")
6367                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6368          VQDMLADHQ_M_S))
6369   ]
6370   "TARGET_HAVE_MVE"
6371   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6372   [(set_attr "type" "mve_move")
6373    (set_attr "length""8")])
6374
6375 ;;
6376 ;; [vqdmladhxq_m_s])
6377 ;;
6378 (define_insn "mve_vqdmladhxq_m_s<mode>"
6379   [
6380    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6381         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6382                        (match_operand:MVE_2 2 "s_register_operand" "w")
6383                        (match_operand:MVE_2 3 "s_register_operand" "w")
6384                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6385          VQDMLADHXQ_M_S))
6386   ]
6387   "TARGET_HAVE_MVE"
6388   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6389   [(set_attr "type" "mve_move")
6390    (set_attr "length""8")])
6391
6392 ;;
6393 ;; [vqdmlsdhq_m_s])
6394 ;;
6395 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6396   [
6397    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6398         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6399                        (match_operand:MVE_2 2 "s_register_operand" "w")
6400                        (match_operand:MVE_2 3 "s_register_operand" "w")
6401                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6402          VQDMLSDHQ_M_S))
6403   ]
6404   "TARGET_HAVE_MVE"
6405   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6406   [(set_attr "type" "mve_move")
6407    (set_attr "length""8")])
6408
6409 ;;
6410 ;; [vqdmlsdhxq_m_s])
6411 ;;
6412 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6413   [
6414    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6415         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6416                        (match_operand:MVE_2 2 "s_register_operand" "w")
6417                        (match_operand:MVE_2 3 "s_register_operand" "w")
6418                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6419          VQDMLSDHXQ_M_S))
6420   ]
6421   "TARGET_HAVE_MVE"
6422   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6423   [(set_attr "type" "mve_move")
6424    (set_attr "length""8")])
6425
6426 ;;
6427 ;; [vqdmulhq_m_n_s])
6428 ;;
6429 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6430   [
6431    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6432         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6433                        (match_operand:MVE_2 2 "s_register_operand" "w")
6434                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6435                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6436          VQDMULHQ_M_N_S))
6437   ]
6438   "TARGET_HAVE_MVE"
6439   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6440   [(set_attr "type" "mve_move")
6441    (set_attr "length""8")])
6442
6443 ;;
6444 ;; [vqdmulhq_m_s])
6445 ;;
6446 (define_insn "mve_vqdmulhq_m_s<mode>"
6447   [
6448    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6449         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6450                        (match_operand:MVE_2 2 "s_register_operand" "w")
6451                        (match_operand:MVE_2 3 "s_register_operand" "w")
6452                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6453          VQDMULHQ_M_S))
6454   ]
6455   "TARGET_HAVE_MVE"
6456   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6457   [(set_attr "type" "mve_move")
6458    (set_attr "length""8")])
6459
6460 ;;
6461 ;; [vqrdmladhq_m_s])
6462 ;;
6463 (define_insn "mve_vqrdmladhq_m_s<mode>"
6464   [
6465    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6466         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6467                        (match_operand:MVE_2 2 "s_register_operand" "w")
6468                        (match_operand:MVE_2 3 "s_register_operand" "w")
6469                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6470          VQRDMLADHQ_M_S))
6471   ]
6472   "TARGET_HAVE_MVE"
6473   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6474   [(set_attr "type" "mve_move")
6475    (set_attr "length""8")])
6476
6477 ;;
6478 ;; [vqrdmladhxq_m_s])
6479 ;;
6480 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6481   [
6482    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6483         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6484                        (match_operand:MVE_2 2 "s_register_operand" "w")
6485                        (match_operand:MVE_2 3 "s_register_operand" "w")
6486                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6487          VQRDMLADHXQ_M_S))
6488   ]
6489   "TARGET_HAVE_MVE"
6490   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6491   [(set_attr "type" "mve_move")
6492    (set_attr "length""8")])
6493
6494 ;;
6495 ;; [vqrdmlsdhq_m_s])
6496 ;;
6497 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6498   [
6499    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6500         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6501                        (match_operand:MVE_2 2 "s_register_operand" "w")
6502                        (match_operand:MVE_2 3 "s_register_operand" "w")
6503                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6504          VQRDMLSDHQ_M_S))
6505   ]
6506   "TARGET_HAVE_MVE"
6507   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6508   [(set_attr "type" "mve_move")
6509    (set_attr "length""8")])
6510
6511 ;;
6512 ;; [vqrdmlsdhxq_m_s])
6513 ;;
6514 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6515   [
6516    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6517         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6518                        (match_operand:MVE_2 2 "s_register_operand" "w")
6519                        (match_operand:MVE_2 3 "s_register_operand" "w")
6520                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6521          VQRDMLSDHXQ_M_S))
6522   ]
6523   "TARGET_HAVE_MVE"
6524   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6525   [(set_attr "type" "mve_move")
6526    (set_attr "length""8")])
6527
6528 ;;
6529 ;; [vqrdmulhq_m_n_s])
6530 ;;
6531 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6532   [
6533    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6534         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6535                        (match_operand:MVE_2 2 "s_register_operand" "w")
6536                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6537                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6538          VQRDMULHQ_M_N_S))
6539   ]
6540   "TARGET_HAVE_MVE"
6541   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6542   [(set_attr "type" "mve_move")
6543    (set_attr "length""8")])
6544
6545 ;;
6546 ;; [vqrdmulhq_m_s])
6547 ;;
6548 (define_insn "mve_vqrdmulhq_m_s<mode>"
6549   [
6550    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6551         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6552                        (match_operand:MVE_2 2 "s_register_operand" "w")
6553                        (match_operand:MVE_2 3 "s_register_operand" "w")
6554                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6555          VQRDMULHQ_M_S))
6556   ]
6557   "TARGET_HAVE_MVE"
6558   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6559   [(set_attr "type" "mve_move")
6560    (set_attr "length""8")])
6561
6562 ;;
6563 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6564 ;;
6565 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6566   [
6567    (set (match_operand:DI 0 "s_register_operand" "=r")
6568         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6569                        (match_operand:MVE_5 2 "s_register_operand" "w")
6570                        (match_operand:MVE_5 3 "s_register_operand" "w")
6571                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6572          VMLALDAVAQ_P))
6573   ]
6574   "TARGET_HAVE_MVE"
6575   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6576   [(set_attr "type" "mve_move")
6577    (set_attr "length""8")])
6578
6579 ;;
6580 ;; [vmlaldavaxq_p_s])
6581 ;;
6582 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6583   [
6584    (set (match_operand:DI 0 "s_register_operand" "=r")
6585         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6586                        (match_operand:MVE_5 2 "s_register_operand" "w")
6587                        (match_operand:MVE_5 3 "s_register_operand" "w")
6588                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6589          VMLALDAVAXQ_P))
6590   ]
6591   "TARGET_HAVE_MVE"
6592   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6593   [(set_attr "type" "mve_move")
6594    (set_attr "length""8")])
6595
6596 ;;
6597 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6598 ;;
6599 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6600   [
6601    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6602         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6603                        (match_operand:MVE_5 2 "s_register_operand" "w")
6604                        (match_operand:SI 3 "mve_imm_8" "Rb")
6605                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6606          VQRSHRNBQ_M_N))
6607   ]
6608   "TARGET_HAVE_MVE"
6609   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6610   [(set_attr "type" "mve_move")
6611    (set_attr "length""8")])
6612
6613 ;;
6614 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6615 ;;
6616 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6617   [
6618    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6619         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6620                        (match_operand:MVE_5 2 "s_register_operand" "w")
6621                        (match_operand:SI 3 "mve_imm_8" "Rb")
6622                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6623          VQRSHRNTQ_M_N))
6624   ]
6625   "TARGET_HAVE_MVE"
6626   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6627   [(set_attr "type" "mve_move")
6628    (set_attr "length""8")])
6629
6630 ;;
6631 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6632 ;;
6633 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6634   [
6635    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6636         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6637                        (match_operand:MVE_5 2 "s_register_operand" "w")
6638                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6639                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6640          VQSHRNBQ_M_N))
6641   ]
6642   "TARGET_HAVE_MVE"
6643   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6644   [(set_attr "type" "mve_move")
6645    (set_attr "length""8")])
6646
6647 ;;
6648 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6649 ;;
6650 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6651   [
6652    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6653         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6654                        (match_operand:MVE_5 2 "s_register_operand" "w")
6655                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6656                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6657          VQSHRNTQ_M_N))
6658   ]
6659   "TARGET_HAVE_MVE"
6660   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6661   [(set_attr "type" "mve_move")
6662    (set_attr "length""8")])
6663
6664 ;;
6665 ;; [vrmlaldavhaq_p_s])
6666 ;;
6667 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6668   [
6669    (set (match_operand:DI 0 "s_register_operand" "=r")
6670         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6671                        (match_operand:V4SI 2 "s_register_operand" "w")
6672                        (match_operand:V4SI 3 "s_register_operand" "w")
6673                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6674          VRMLALDAVHAQ_P_S))
6675   ]
6676   "TARGET_HAVE_MVE"
6677   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6678   [(set_attr "type" "mve_move")
6679    (set_attr "length""8")])
6680
6681 ;;
6682 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6683 ;;
6684 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6685   [
6686    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6687         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6688                        (match_operand:MVE_5 2 "s_register_operand" "w")
6689                        (match_operand:SI 3 "mve_imm_8" "Rb")
6690                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6691          VRSHRNBQ_M_N))
6692   ]
6693   "TARGET_HAVE_MVE"
6694   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6695   [(set_attr "type" "mve_move")
6696    (set_attr "length""8")])
6697
6698 ;;
6699 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6700 ;;
6701 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6702   [
6703    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6704         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6705                        (match_operand:MVE_5 2 "s_register_operand" "w")
6706                        (match_operand:SI 3 "mve_imm_8" "Rb")
6707                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6708          VRSHRNTQ_M_N))
6709   ]
6710   "TARGET_HAVE_MVE"
6711   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6712   [(set_attr "type" "mve_move")
6713    (set_attr "length""8")])
6714
6715 ;;
6716 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6717 ;;
6718 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6719   [
6720    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6721         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6722                        (match_operand:MVE_3 2 "s_register_operand" "w")
6723                        (match_operand:SI 3 "immediate_operand" "i")
6724                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6725          VSHLLBQ_M_N))
6726   ]
6727   "TARGET_HAVE_MVE"
6728   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6729   [(set_attr "type" "mve_move")
6730    (set_attr "length""8")])
6731
6732 ;;
6733 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6734 ;;
6735 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6736   [
6737    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6738         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6739                        (match_operand:MVE_3 2 "s_register_operand" "w")
6740                        (match_operand:SI 3 "immediate_operand" "i")
6741                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6742          VSHLLTQ_M_N))
6743   ]
6744   "TARGET_HAVE_MVE"
6745   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6746   [(set_attr "type" "mve_move")
6747    (set_attr "length""8")])
6748
6749 ;;
6750 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6751 ;;
6752 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6753   [
6754    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6755         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6756                        (match_operand:MVE_5 2 "s_register_operand" "w")
6757                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6758                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6759          VSHRNBQ_M_N))
6760   ]
6761   "TARGET_HAVE_MVE"
6762   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6763   [(set_attr "type" "mve_move")
6764    (set_attr "length""8")])
6765
6766 ;;
6767 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6768 ;;
6769 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6770   [
6771    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6772         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6773                        (match_operand:MVE_5 2 "s_register_operand" "w")
6774                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6775                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6776          VSHRNTQ_M_N))
6777   ]
6778   "TARGET_HAVE_MVE"
6779   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6780   [(set_attr "type" "mve_move")
6781    (set_attr "length""8")])
6782
6783 ;;
6784 ;; [vmlsldavaq_p_s])
6785 ;;
6786 (define_insn "mve_vmlsldavaq_p_s<mode>"
6787   [
6788    (set (match_operand:DI 0 "s_register_operand" "=r")
6789         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6790                        (match_operand:MVE_5 2 "s_register_operand" "w")
6791                        (match_operand:MVE_5 3 "s_register_operand" "w")
6792                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6793          VMLSLDAVAQ_P_S))
6794   ]
6795   "TARGET_HAVE_MVE"
6796   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6797   [(set_attr "type" "mve_move")
6798    (set_attr "length""8")])
6799
6800 ;;
6801 ;; [vmlsldavaxq_p_s])
6802 ;;
6803 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6804   [
6805    (set (match_operand:DI 0 "s_register_operand" "=r")
6806         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6807                        (match_operand:MVE_5 2 "s_register_operand" "w")
6808                        (match_operand:MVE_5 3 "s_register_operand" "w")
6809                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6810          VMLSLDAVAXQ_P_S))
6811   ]
6812   "TARGET_HAVE_MVE"
6813   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6814   [(set_attr "type" "mve_move")
6815    (set_attr "length""8")])
6816
6817 ;;
6818 ;; [vmullbq_poly_m_p])
6819 ;;
6820 (define_insn "mve_vmullbq_poly_m_p<mode>"
6821   [
6822    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6823         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6824                        (match_operand:MVE_3 2 "s_register_operand" "w")
6825                        (match_operand:MVE_3 3 "s_register_operand" "w")
6826                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6827          VMULLBQ_POLY_M_P))
6828   ]
6829   "TARGET_HAVE_MVE"
6830   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6831   [(set_attr "type" "mve_move")
6832    (set_attr "length""8")])
6833
6834 ;;
6835 ;; [vmulltq_poly_m_p])
6836 ;;
6837 (define_insn "mve_vmulltq_poly_m_p<mode>"
6838   [
6839    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6840         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6841                        (match_operand:MVE_3 2 "s_register_operand" "w")
6842                        (match_operand:MVE_3 3 "s_register_operand" "w")
6843                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6844          VMULLTQ_POLY_M_P))
6845   ]
6846   "TARGET_HAVE_MVE"
6847   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6848   [(set_attr "type" "mve_move")
6849    (set_attr "length""8")])
6850
6851 ;;
6852 ;; [vqdmullbq_m_n_s])
6853 ;;
6854 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6855   [
6856    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6857         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6858                        (match_operand:MVE_5 2 "s_register_operand" "w")
6859                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6860                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6861          VQDMULLBQ_M_N_S))
6862   ]
6863   "TARGET_HAVE_MVE"
6864   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6865   [(set_attr "type" "mve_move")
6866    (set_attr "length""8")])
6867
6868 ;;
6869 ;; [vqdmullbq_m_s])
6870 ;;
6871 (define_insn "mve_vqdmullbq_m_s<mode>"
6872   [
6873    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6874         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6875                        (match_operand:MVE_5 2 "s_register_operand" "w")
6876                        (match_operand:MVE_5 3 "s_register_operand" "w")
6877                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6878          VQDMULLBQ_M_S))
6879   ]
6880   "TARGET_HAVE_MVE"
6881   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6882   [(set_attr "type" "mve_move")
6883    (set_attr "length""8")])
6884
6885 ;;
6886 ;; [vqdmulltq_m_n_s])
6887 ;;
6888 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6889   [
6890    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6891         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6892                        (match_operand:MVE_5 2 "s_register_operand" "w")
6893                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6894                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6895          VQDMULLTQ_M_N_S))
6896   ]
6897   "TARGET_HAVE_MVE"
6898   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6899   [(set_attr "type" "mve_move")
6900    (set_attr "length""8")])
6901
6902 ;;
6903 ;; [vqdmulltq_m_s])
6904 ;;
6905 (define_insn "mve_vqdmulltq_m_s<mode>"
6906   [
6907    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6908         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6909                        (match_operand:MVE_5 2 "s_register_operand" "w")
6910                        (match_operand:MVE_5 3 "s_register_operand" "w")
6911                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6912          VQDMULLTQ_M_S))
6913   ]
6914   "TARGET_HAVE_MVE"
6915   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6916   [(set_attr "type" "mve_move")
6917    (set_attr "length""8")])
6918
6919 ;;
6920 ;; [vqrshrunbq_m_n_s])
6921 ;;
6922 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6923   [
6924    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6925         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6926                        (match_operand:MVE_5 2 "s_register_operand" "w")
6927                        (match_operand:SI 3 "mve_imm_8" "Rb")
6928                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6929          VQRSHRUNBQ_M_N_S))
6930   ]
6931   "TARGET_HAVE_MVE"
6932   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6933   [(set_attr "type" "mve_move")
6934    (set_attr "length""8")])
6935
6936 ;;
6937 ;; [vqrshruntq_m_n_s])
6938 ;;
6939 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6940   [
6941    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6942         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6943                        (match_operand:MVE_5 2 "s_register_operand" "w")
6944                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6945                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6946          VQRSHRUNTQ_M_N_S))
6947   ]
6948   "TARGET_HAVE_MVE"
6949   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6950   [(set_attr "type" "mve_move")
6951    (set_attr "length""8")])
6952
6953 ;;
6954 ;; [vqshrunbq_m_n_s])
6955 ;;
6956 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6957   [
6958    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6959         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6960                        (match_operand:MVE_5 2 "s_register_operand" "w")
6961                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6962                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6963          VQSHRUNBQ_M_N_S))
6964   ]
6965   "TARGET_HAVE_MVE"
6966   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6967   [(set_attr "type" "mve_move")
6968    (set_attr "length""8")])
6969
6970 ;;
6971 ;; [vqshruntq_m_n_s])
6972 ;;
6973 (define_insn "mve_vqshruntq_m_n_s<mode>"
6974   [
6975    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6976         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6977                        (match_operand:MVE_5 2 "s_register_operand" "w")
6978                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6979                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6980          VQSHRUNTQ_M_N_S))
6981   ]
6982   "TARGET_HAVE_MVE"
6983   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6984   [(set_attr "type" "mve_move")
6985    (set_attr "length""8")])
6986
6987 ;;
6988 ;; [vrmlaldavhaq_p_u])
6989 ;;
6990 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6991   [
6992    (set (match_operand:DI 0 "s_register_operand" "=r")
6993         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6994                        (match_operand:V4SI 2 "s_register_operand" "w")
6995                        (match_operand:V4SI 3 "s_register_operand" "w")
6996                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6997          VRMLALDAVHAQ_P_U))
6998   ]
6999   "TARGET_HAVE_MVE"
7000   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7001   [(set_attr "type" "mve_move")
7002    (set_attr "length""8")])
7003
7004 ;;
7005 ;; [vrmlaldavhaxq_p_s])
7006 ;;
7007 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7008   [
7009    (set (match_operand:DI 0 "s_register_operand" "=r")
7010         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7011                        (match_operand:V4SI 2 "s_register_operand" "w")
7012                        (match_operand:V4SI 3 "s_register_operand" "w")
7013                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7014          VRMLALDAVHAXQ_P_S))
7015   ]
7016   "TARGET_HAVE_MVE"
7017   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7018   [(set_attr "type" "mve_move")
7019    (set_attr "length""8")])
7020
7021 ;;
7022 ;; [vrmlsldavhaq_p_s])
7023 ;;
7024 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7025   [
7026    (set (match_operand:DI 0 "s_register_operand" "=r")
7027         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7028                        (match_operand:V4SI 2 "s_register_operand" "w")
7029                        (match_operand:V4SI 3 "s_register_operand" "w")
7030                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7031          VRMLSLDAVHAQ_P_S))
7032   ]
7033   "TARGET_HAVE_MVE"
7034   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7035   [(set_attr "type" "mve_move")
7036    (set_attr "length""8")])
7037
7038 ;;
7039 ;; [vrmlsldavhaxq_p_s])
7040 ;;
7041 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7042   [
7043    (set (match_operand:DI 0 "s_register_operand" "=r")
7044         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7045                        (match_operand:V4SI 2 "s_register_operand" "w")
7046                        (match_operand:V4SI 3 "s_register_operand" "w")
7047                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7048          VRMLSLDAVHAXQ_P_S))
7049   ]
7050   "TARGET_HAVE_MVE"
7051   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7052   [(set_attr "type" "mve_move")
7053    (set_attr "length""8")])
7054 ;;
7055 ;; [vabdq_m_f])
7056 ;;
7057 (define_insn "mve_vabdq_m_f<mode>"
7058   [
7059    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7060         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7061                        (match_operand:MVE_0 2 "s_register_operand" "w")
7062                        (match_operand:MVE_0 3 "s_register_operand" "w")
7063                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7064          VABDQ_M_F))
7065   ]
7066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7067   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7068   [(set_attr "type" "mve_move")
7069    (set_attr "length""8")])
7070
7071 ;;
7072 ;; [vaddq_m_f])
7073 ;;
7074 (define_insn "mve_vaddq_m_f<mode>"
7075   [
7076    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7077         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7078                        (match_operand:MVE_0 2 "s_register_operand" "w")
7079                        (match_operand:MVE_0 3 "s_register_operand" "w")
7080                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7081          VADDQ_M_F))
7082   ]
7083   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7084   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7085   [(set_attr "type" "mve_move")
7086    (set_attr "length""8")])
7087
7088 ;;
7089 ;; [vaddq_m_n_f])
7090 ;;
7091 (define_insn "mve_vaddq_m_n_f<mode>"
7092   [
7093    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7094         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7095                        (match_operand:MVE_0 2 "s_register_operand" "w")
7096                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7097                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7098          VADDQ_M_N_F))
7099   ]
7100   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7101   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7102   [(set_attr "type" "mve_move")
7103    (set_attr "length""8")])
7104
7105 ;;
7106 ;; [vandq_m_f])
7107 ;;
7108 (define_insn "mve_vandq_m_f<mode>"
7109   [
7110    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7111         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7112                        (match_operand:MVE_0 2 "s_register_operand" "w")
7113                        (match_operand:MVE_0 3 "s_register_operand" "w")
7114                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7115          VANDQ_M_F))
7116   ]
7117   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7118   "vpst\;vandt %q0, %q2, %q3"
7119   [(set_attr "type" "mve_move")
7120    (set_attr "length""8")])
7121
7122 ;;
7123 ;; [vbicq_m_f])
7124 ;;
7125 (define_insn "mve_vbicq_m_f<mode>"
7126   [
7127    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7128         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7129                        (match_operand:MVE_0 2 "s_register_operand" "w")
7130                        (match_operand:MVE_0 3 "s_register_operand" "w")
7131                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7132          VBICQ_M_F))
7133   ]
7134   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7135   "vpst\;vbict %q0, %q2, %q3"
7136   [(set_attr "type" "mve_move")
7137    (set_attr "length""8")])
7138
7139 ;;
7140 ;; [vbrsrq_m_n_f])
7141 ;;
7142 (define_insn "mve_vbrsrq_m_n_f<mode>"
7143   [
7144    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7145         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7146                        (match_operand:MVE_0 2 "s_register_operand" "w")
7147                        (match_operand:SI 3 "s_register_operand" "r")
7148                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7149          VBRSRQ_M_N_F))
7150   ]
7151   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7152   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7153   [(set_attr "type" "mve_move")
7154    (set_attr "length""8")])
7155
7156 ;;
7157 ;; [vcaddq_rot270_m_f])
7158 ;;
7159 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7160   [
7161    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7162         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7163                        (match_operand:MVE_0 2 "s_register_operand" "w")
7164                        (match_operand:MVE_0 3 "s_register_operand" "w")
7165                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7166          VCADDQ_ROT270_M_F))
7167   ]
7168   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7169   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7170   [(set_attr "type" "mve_move")
7171    (set_attr "length""8")])
7172
7173 ;;
7174 ;; [vcaddq_rot90_m_f])
7175 ;;
7176 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7177   [
7178    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7179         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7180                        (match_operand:MVE_0 2 "s_register_operand" "w")
7181                        (match_operand:MVE_0 3 "s_register_operand" "w")
7182                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7183          VCADDQ_ROT90_M_F))
7184   ]
7185   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7186   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7187   [(set_attr "type" "mve_move")
7188    (set_attr "length""8")])
7189
7190 ;;
7191 ;; [vcmlaq_m_f])
7192 ;;
7193 (define_insn "mve_vcmlaq_m_f<mode>"
7194   [
7195    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7196         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7197                        (match_operand:MVE_0 2 "s_register_operand" "w")
7198                        (match_operand:MVE_0 3 "s_register_operand" "w")
7199                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7200          VCMLAQ_M_F))
7201   ]
7202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7203   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7204   [(set_attr "type" "mve_move")
7205    (set_attr "length""8")])
7206
7207 ;;
7208 ;; [vcmlaq_rot180_m_f])
7209 ;;
7210 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7211   [
7212    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7213         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7214                        (match_operand:MVE_0 2 "s_register_operand" "w")
7215                        (match_operand:MVE_0 3 "s_register_operand" "w")
7216                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7217          VCMLAQ_ROT180_M_F))
7218   ]
7219   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7220   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7221   [(set_attr "type" "mve_move")
7222    (set_attr "length""8")])
7223
7224 ;;
7225 ;; [vcmlaq_rot270_m_f])
7226 ;;
7227 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7228   [
7229    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7230         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7231                        (match_operand:MVE_0 2 "s_register_operand" "w")
7232                        (match_operand:MVE_0 3 "s_register_operand" "w")
7233                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7234          VCMLAQ_ROT270_M_F))
7235   ]
7236   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7237   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7238   [(set_attr "type" "mve_move")
7239    (set_attr "length""8")])
7240
7241 ;;
7242 ;; [vcmlaq_rot90_m_f])
7243 ;;
7244 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7245   [
7246    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7247         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7248                        (match_operand:MVE_0 2 "s_register_operand" "w")
7249                        (match_operand:MVE_0 3 "s_register_operand" "w")
7250                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7251          VCMLAQ_ROT90_M_F))
7252   ]
7253   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7254   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7255   [(set_attr "type" "mve_move")
7256    (set_attr "length""8")])
7257
7258 ;;
7259 ;; [vcmulq_m_f])
7260 ;;
7261 (define_insn "mve_vcmulq_m_f<mode>"
7262   [
7263    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7264         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7265                        (match_operand:MVE_0 2 "s_register_operand" "w")
7266                        (match_operand:MVE_0 3 "s_register_operand" "w")
7267                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7268          VCMULQ_M_F))
7269   ]
7270   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7271   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7272   [(set_attr "type" "mve_move")
7273    (set_attr "length""8")])
7274
7275 ;;
7276 ;; [vcmulq_rot180_m_f])
7277 ;;
7278 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7279   [
7280    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7281         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7282                        (match_operand:MVE_0 2 "s_register_operand" "w")
7283                        (match_operand:MVE_0 3 "s_register_operand" "w")
7284                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7285          VCMULQ_ROT180_M_F))
7286   ]
7287   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7288   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7289   [(set_attr "type" "mve_move")
7290    (set_attr "length""8")])
7291
7292 ;;
7293 ;; [vcmulq_rot270_m_f])
7294 ;;
7295 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7296   [
7297    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7298         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7299                        (match_operand:MVE_0 2 "s_register_operand" "w")
7300                        (match_operand:MVE_0 3 "s_register_operand" "w")
7301                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7302          VCMULQ_ROT270_M_F))
7303   ]
7304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7305   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7306   [(set_attr "type" "mve_move")
7307    (set_attr "length""8")])
7308
7309 ;;
7310 ;; [vcmulq_rot90_m_f])
7311 ;;
7312 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7313   [
7314    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7315         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7316                        (match_operand:MVE_0 2 "s_register_operand" "w")
7317                        (match_operand:MVE_0 3 "s_register_operand" "w")
7318                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7319          VCMULQ_ROT90_M_F))
7320   ]
7321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7322   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7323   [(set_attr "type" "mve_move")
7324    (set_attr "length""8")])
7325
7326 ;;
7327 ;; [veorq_m_f])
7328 ;;
7329 (define_insn "mve_veorq_m_f<mode>"
7330   [
7331    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7332         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7333                        (match_operand:MVE_0 2 "s_register_operand" "w")
7334                        (match_operand:MVE_0 3 "s_register_operand" "w")
7335                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7336          VEORQ_M_F))
7337   ]
7338   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7339   "vpst\;veort %q0, %q2, %q3"
7340   [(set_attr "type" "mve_move")
7341    (set_attr "length""8")])
7342
7343 ;;
7344 ;; [vfmaq_m_f])
7345 ;;
7346 (define_insn "mve_vfmaq_m_f<mode>"
7347   [
7348    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7349         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7350                        (match_operand:MVE_0 2 "s_register_operand" "w")
7351                        (match_operand:MVE_0 3 "s_register_operand" "w")
7352                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7353          VFMAQ_M_F))
7354   ]
7355   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7356   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7357   [(set_attr "type" "mve_move")
7358    (set_attr "length""8")])
7359
7360 ;;
7361 ;; [vfmaq_m_n_f])
7362 ;;
7363 (define_insn "mve_vfmaq_m_n_f<mode>"
7364   [
7365    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7366         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7367                        (match_operand:MVE_0 2 "s_register_operand" "w")
7368                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7369                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7370          VFMAQ_M_N_F))
7371   ]
7372   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7373   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7374   [(set_attr "type" "mve_move")
7375    (set_attr "length""8")])
7376
7377 ;;
7378 ;; [vfmasq_m_n_f])
7379 ;;
7380 (define_insn "mve_vfmasq_m_n_f<mode>"
7381   [
7382    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7383         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7384                        (match_operand:MVE_0 2 "s_register_operand" "w")
7385                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7386                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7387          VFMASQ_M_N_F))
7388   ]
7389   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7390   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7391   [(set_attr "type" "mve_move")
7392    (set_attr "length""8")])
7393
7394 ;;
7395 ;; [vfmsq_m_f])
7396 ;;
7397 (define_insn "mve_vfmsq_m_f<mode>"
7398   [
7399    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7400         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7401                        (match_operand:MVE_0 2 "s_register_operand" "w")
7402                        (match_operand:MVE_0 3 "s_register_operand" "w")
7403                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7404          VFMSQ_M_F))
7405   ]
7406   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7407   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7408   [(set_attr "type" "mve_move")
7409    (set_attr "length""8")])
7410
7411 ;;
7412 ;; [vmaxnmq_m_f])
7413 ;;
7414 (define_insn "mve_vmaxnmq_m_f<mode>"
7415   [
7416    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7417         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7418                        (match_operand:MVE_0 2 "s_register_operand" "w")
7419                        (match_operand:MVE_0 3 "s_register_operand" "w")
7420                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7421          VMAXNMQ_M_F))
7422   ]
7423   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7424   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7425   [(set_attr "type" "mve_move")
7426    (set_attr "length""8")])
7427
7428 ;;
7429 ;; [vminnmq_m_f])
7430 ;;
7431 (define_insn "mve_vminnmq_m_f<mode>"
7432   [
7433    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7434         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7435                        (match_operand:MVE_0 2 "s_register_operand" "w")
7436                        (match_operand:MVE_0 3 "s_register_operand" "w")
7437                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7438          VMINNMQ_M_F))
7439   ]
7440   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7441   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7442   [(set_attr "type" "mve_move")
7443    (set_attr "length""8")])
7444
7445 ;;
7446 ;; [vmulq_m_f])
7447 ;;
7448 (define_insn "mve_vmulq_m_f<mode>"
7449   [
7450    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7451         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7452                        (match_operand:MVE_0 2 "s_register_operand" "w")
7453                        (match_operand:MVE_0 3 "s_register_operand" "w")
7454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7455          VMULQ_M_F))
7456   ]
7457   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7458   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7459   [(set_attr "type" "mve_move")
7460    (set_attr "length""8")])
7461
7462 ;;
7463 ;; [vmulq_m_n_f])
7464 ;;
7465 (define_insn "mve_vmulq_m_n_f<mode>"
7466   [
7467    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7468         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7469                        (match_operand:MVE_0 2 "s_register_operand" "w")
7470                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7471                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7472          VMULQ_M_N_F))
7473   ]
7474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7475   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7476   [(set_attr "type" "mve_move")
7477    (set_attr "length""8")])
7478
7479 ;;
7480 ;; [vornq_m_f])
7481 ;;
7482 (define_insn "mve_vornq_m_f<mode>"
7483   [
7484    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7485         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7486                        (match_operand:MVE_0 2 "s_register_operand" "w")
7487                        (match_operand:MVE_0 3 "s_register_operand" "w")
7488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7489          VORNQ_M_F))
7490   ]
7491   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7492   "vpst\;vornt %q0, %q2, %q3"
7493   [(set_attr "type" "mve_move")
7494    (set_attr "length""8")])
7495
7496 ;;
7497 ;; [vorrq_m_f])
7498 ;;
7499 (define_insn "mve_vorrq_m_f<mode>"
7500   [
7501    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7502         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7503                        (match_operand:MVE_0 2 "s_register_operand" "w")
7504                        (match_operand:MVE_0 3 "s_register_operand" "w")
7505                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7506          VORRQ_M_F))
7507   ]
7508   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7509   "vpst\;vorrt %q0, %q2, %q3"
7510   [(set_attr "type" "mve_move")
7511    (set_attr "length""8")])
7512
7513 ;;
7514 ;; [vsubq_m_f])
7515 ;;
7516 (define_insn "mve_vsubq_m_f<mode>"
7517   [
7518    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7519         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7520                        (match_operand:MVE_0 2 "s_register_operand" "w")
7521                        (match_operand:MVE_0 3 "s_register_operand" "w")
7522                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7523          VSUBQ_M_F))
7524   ]
7525   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7526   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7527   [(set_attr "type" "mve_move")
7528    (set_attr "length""8")])
7529
7530 ;;
7531 ;; [vsubq_m_n_f])
7532 ;;
7533 (define_insn "mve_vsubq_m_n_f<mode>"
7534   [
7535    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7536         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7537                        (match_operand:MVE_0 2 "s_register_operand" "w")
7538                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7539                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7540          VSUBQ_M_N_F))
7541   ]
7542   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7543   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7544   [(set_attr "type" "mve_move")
7545    (set_attr "length""8")])
7546
7547 ;;
7548 ;; [vstrbq_s vstrbq_u]
7549 ;;
7550 (define_insn "mve_vstrbq_<supf><mode>"
7551   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7552         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7553          VSTRBQ))
7554   ]
7555   "TARGET_HAVE_MVE"
7556 {
7557    rtx ops[2];
7558    int regno = REGNO (operands[1]);
7559    ops[1] = gen_rtx_REG (TImode, regno);
7560    ops[0]  = operands[0];
7561    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7562    return "";
7563 }
7564   [(set_attr "length" "4")])
7565
7566 ;;
7567 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7568 ;;
7569 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7570   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7571    (match_operand:MVE_2 1 "s_register_operand")
7572    (match_operand:MVE_2 2 "s_register_operand")
7573    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7574   "TARGET_HAVE_MVE"
7575 {
7576   rtx ind = XEXP (operands[0], 0);
7577   gcc_assert (REG_P (ind));
7578   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7579                                                               operands[2]));
7580   DONE;
7581 })
7582
7583 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7584   [(set (mem:BLK (scratch))
7585         (unspec:BLK
7586           [(match_operand:SI 0 "register_operand" "r")
7587            (match_operand:MVE_2 1 "s_register_operand" "w")
7588            (match_operand:MVE_2 2 "s_register_operand" "w")]
7589           VSTRBSOQ))]
7590   "TARGET_HAVE_MVE"
7591   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7592   [(set_attr "length" "4")])
7593
7594 ;;
7595 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7596 ;;
7597 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7598   [(set (mem:BLK (scratch))
7599         (unspec:BLK
7600                 [(match_operand:V4SI 0 "s_register_operand" "w")
7601                  (match_operand:SI 1 "immediate_operand" "i")
7602                  (match_operand:V4SI 2 "s_register_operand" "w")]
7603          VSTRWSBQ))
7604   ]
7605   "TARGET_HAVE_MVE"
7606 {
7607    rtx ops[3];
7608    ops[0] = operands[0];
7609    ops[1] = operands[1];
7610    ops[2] = operands[2];
7611    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7612    return "";
7613 }
7614   [(set_attr "length" "4")])
7615
7616 ;;
7617 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7618 ;;
7619 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7620   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7621         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7622                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7623          VLDRBGOQ))
7624   ]
7625   "TARGET_HAVE_MVE"
7626 {
7627    rtx ops[3];
7628    ops[0] = operands[0];
7629    ops[1] = operands[1];
7630    ops[2] = operands[2];
7631    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7632      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7633    else
7634      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7635    return "";
7636 }
7637   [(set_attr "length" "4")])
7638
7639 ;;
7640 ;; [vldrbq_s vldrbq_u]
7641 ;;
7642 (define_insn "mve_vldrbq_<supf><mode>"
7643   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7644         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7645          VLDRBQ))
7646   ]
7647   "TARGET_HAVE_MVE"
7648 {
7649    rtx ops[2];
7650    int regno = REGNO (operands[0]);
7651    ops[0] = gen_rtx_REG (TImode, regno);
7652    ops[1]  = operands[1];
7653    if (<V_sz_elem> == 8)
7654      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7655    else
7656      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7657    return "";
7658 }
7659   [(set_attr "length" "4")])
7660
7661 ;;
7662 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7663 ;;
7664 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7665   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7666         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7667                       (match_operand:SI 2 "immediate_operand" "i")]
7668          VLDRWGBQ))
7669   ]
7670   "TARGET_HAVE_MVE"
7671 {
7672    rtx ops[3];
7673    ops[0] = operands[0];
7674    ops[1] = operands[1];
7675    ops[2] = operands[2];
7676    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7677    return "";
7678 }
7679   [(set_attr "length" "4")])
7680
7681 ;;
7682 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7683 ;;
7684 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7685   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7686    (match_operand:MVE_2 1 "s_register_operand")
7687    (match_operand:MVE_2 2 "s_register_operand")
7688    (match_operand:HI 3 "vpr_register_operand" "Up")
7689    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7690   "TARGET_HAVE_MVE"
7691 {
7692   rtx ind = XEXP (operands[0], 0);
7693   gcc_assert (REG_P (ind));
7694   emit_insn (
7695     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7696                                                        operands[2],
7697                                                        operands[3]));
7698   DONE;
7699 })
7700
7701 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7702   [(set (mem:BLK (scratch))
7703         (unspec:BLK
7704           [(match_operand:SI 0 "register_operand" "r")
7705            (match_operand:MVE_2 1 "s_register_operand" "w")
7706            (match_operand:MVE_2 2 "s_register_operand" "w")
7707            (match_operand:HI 3 "vpr_register_operand" "Up")]
7708           VSTRBSOQ))]
7709   "TARGET_HAVE_MVE"
7710   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7711   [(set_attr "length" "8")])
7712
7713 ;;
7714 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7715 ;;
7716 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7717   [(set (mem:BLK (scratch))
7718         (unspec:BLK
7719                 [(match_operand:V4SI 0 "s_register_operand" "w")
7720                  (match_operand:SI 1 "immediate_operand" "i")
7721                  (match_operand:V4SI 2 "s_register_operand" "w")
7722                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7723          VSTRWSBQ))
7724   ]
7725   "TARGET_HAVE_MVE"
7726 {
7727    rtx ops[3];
7728    ops[0] = operands[0];
7729    ops[1] = operands[1];
7730    ops[2] = operands[2];
7731    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7732    return "";
7733 }
7734   [(set_attr "length" "8")])
7735
7736 ;;
7737 ;; [vstrbq_p_s vstrbq_p_u]
7738 ;;
7739 (define_insn "mve_vstrbq_p_<supf><mode>"
7740   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7741         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7742                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7743          VSTRBQ))
7744   ]
7745   "TARGET_HAVE_MVE"
7746 {
7747    rtx ops[2];
7748    int regno = REGNO (operands[1]);
7749    ops[1] = gen_rtx_REG (TImode, regno);
7750    ops[0]  = operands[0];
7751    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7752    return "";
7753 }
7754   [(set_attr "length" "8")])
7755
7756 ;;
7757 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7758 ;;
7759 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7760   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7761         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7762                        (match_operand:MVE_2 2 "s_register_operand" "w")
7763                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7764          VLDRBGOQ))
7765   ]
7766   "TARGET_HAVE_MVE"
7767 {
7768    rtx ops[4];
7769    ops[0] = operands[0];
7770    ops[1] = operands[1];
7771    ops[2] = operands[2];
7772    ops[3] = operands[3];
7773    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7774      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7775    else
7776      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7777    return "";
7778 }
7779   [(set_attr "length" "8")])
7780
7781 ;;
7782 ;; [vldrbq_z_s vldrbq_z_u]
7783 ;;
7784 (define_insn "mve_vldrbq_z_<supf><mode>"
7785   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7786         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7787                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7788          VLDRBQ))
7789   ]
7790   "TARGET_HAVE_MVE"
7791 {
7792    rtx ops[2];
7793    int regno = REGNO (operands[0]);
7794    ops[0] = gen_rtx_REG (TImode, regno);
7795    ops[1]  = operands[1];
7796    if (<V_sz_elem> == 8)
7797      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7798    else
7799      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7800    return "";
7801 }
7802   [(set_attr "length" "8")])
7803
7804 ;;
7805 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7806 ;;
7807 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7808   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7809         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7810                       (match_operand:SI 2 "immediate_operand" "i")
7811                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7812          VLDRWGBQ))
7813   ]
7814   "TARGET_HAVE_MVE"
7815 {
7816    rtx ops[3];
7817    ops[0] = operands[0];
7818    ops[1] = operands[1];
7819    ops[2] = operands[2];
7820    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7821    return "";
7822 }
7823   [(set_attr "length" "8")])
7824
7825 ;;
7826 ;; [vldrhq_f]
7827 ;;
7828 (define_insn "mve_vldrhq_fv8hf"
7829   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7830         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7831          VLDRHQ_F))
7832   ]
7833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7834 {
7835    rtx ops[2];
7836    int regno = REGNO (operands[0]);
7837    ops[0] = gen_rtx_REG (TImode, regno);
7838    ops[1]  = operands[1];
7839    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7840    return "";
7841 }
7842   [(set_attr "length" "4")])
7843
7844 ;;
7845 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7846 ;;
7847 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7848   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7849         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7850                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7851         VLDRHGOQ))
7852   ]
7853   "TARGET_HAVE_MVE"
7854 {
7855    rtx ops[3];
7856    ops[0] = operands[0];
7857    ops[1] = operands[1];
7858    ops[2] = operands[2];
7859    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7860      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7861    else
7862      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7863    return "";
7864 }
7865   [(set_attr "length" "4")])
7866
7867 ;;
7868 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7869 ;;
7870 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7871   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7872         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7873                        (match_operand:MVE_6 2 "s_register_operand" "w")
7874                        (match_operand:HI 3 "vpr_register_operand" "Up")
7875         ]VLDRHGOQ))
7876   ]
7877   "TARGET_HAVE_MVE"
7878 {
7879    rtx ops[4];
7880    ops[0] = operands[0];
7881    ops[1] = operands[1];
7882    ops[2] = operands[2];
7883    ops[3] = operands[3];
7884    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7885      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7886    else
7887      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7888    return "";
7889 }
7890  [(set_attr "length" "8")])
7891
7892 ;;
7893 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7894 ;;
7895 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7896   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7897         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7898                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7899         VLDRHGSOQ))
7900   ]
7901   "TARGET_HAVE_MVE"
7902 {
7903    rtx ops[3];
7904    ops[0] = operands[0];
7905    ops[1] = operands[1];
7906    ops[2] = operands[2];
7907       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7908      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7909    else
7910      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7911    return "";
7912 }
7913   [(set_attr "length" "4")])
7914
7915 ;;
7916 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7917 ;;
7918 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7919   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7920         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7921                        (match_operand:MVE_6 2 "s_register_operand" "w")
7922                        (match_operand:HI 3 "vpr_register_operand" "Up")
7923         ]VLDRHGSOQ))
7924   ]
7925   "TARGET_HAVE_MVE"
7926 {
7927    rtx ops[4];
7928    ops[0] = operands[0];
7929    ops[1] = operands[1];
7930    ops[2] = operands[2];
7931    ops[3] = operands[3];
7932    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7933      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7934    else
7935      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7936    return "";
7937 }
7938   [(set_attr "length" "8")])
7939
7940 ;;
7941 ;; [vldrhq_s, vldrhq_u]
7942 ;;
7943 (define_insn "mve_vldrhq_<supf><mode>"
7944   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7945         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7946          VLDRHQ))
7947   ]
7948   "TARGET_HAVE_MVE"
7949 {
7950    rtx ops[2];
7951    int regno = REGNO (operands[0]);
7952    ops[0] = gen_rtx_REG (TImode, regno);
7953    ops[1]  = operands[1];
7954    if (<V_sz_elem> == 16)
7955      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7956    else
7957      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7958    return "";
7959 }
7960   [(set_attr "length" "4")])
7961
7962 ;;
7963 ;; [vldrhq_z_f]
7964 ;;
7965 (define_insn "mve_vldrhq_z_fv8hf"
7966   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7967         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7968         (match_operand:HI 2 "vpr_register_operand" "Up")]
7969          VLDRHQ_F))
7970   ]
7971   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7972 {
7973    rtx ops[2];
7974    int regno = REGNO (operands[0]);
7975    ops[0] = gen_rtx_REG (TImode, regno);
7976    ops[1]  = operands[1];
7977    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7978    return "";
7979 }
7980   [(set_attr "length" "8")])
7981
7982 ;;
7983 ;; [vldrhq_z_s vldrhq_z_u]
7984 ;;
7985 (define_insn "mve_vldrhq_z_<supf><mode>"
7986   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7987         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7988         (match_operand:HI 2 "vpr_register_operand" "Up")]
7989          VLDRHQ))
7990   ]
7991   "TARGET_HAVE_MVE"
7992 {
7993    rtx ops[2];
7994    int regno = REGNO (operands[0]);
7995    ops[0] = gen_rtx_REG (TImode, regno);
7996    ops[1]  = operands[1];
7997    if (<V_sz_elem> == 16)
7998      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7999    else
8000      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8001    return "";
8002 }
8003   [(set_attr "length" "8")])
8004
8005 ;;
8006 ;; [vldrwq_f]
8007 ;;
8008 (define_insn "mve_vldrwq_fv4sf"
8009   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8010         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
8011          VLDRWQ_F))
8012   ]
8013   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8014 {
8015    rtx ops[2];
8016    int regno = REGNO (operands[0]);
8017    ops[0] = gen_rtx_REG (TImode, regno);
8018    ops[1]  = operands[1];
8019    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8020    return "";
8021 }
8022   [(set_attr "length" "4")])
8023
8024 ;;
8025 ;; [vldrwq_s vldrwq_u]
8026 ;;
8027 (define_insn "mve_vldrwq_<supf>v4si"
8028   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8029         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
8030          VLDRWQ))
8031   ]
8032   "TARGET_HAVE_MVE"
8033 {
8034    rtx ops[2];
8035    int regno = REGNO (operands[0]);
8036    ops[0] = gen_rtx_REG (TImode, regno);
8037    ops[1]  = operands[1];
8038    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8039    return "";
8040 }
8041   [(set_attr "length" "4")])
8042
8043 ;;
8044 ;; [vldrwq_z_f]
8045 ;;
8046 (define_insn "mve_vldrwq_z_fv4sf"
8047   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8048         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
8049         (match_operand:HI 2 "vpr_register_operand" "Up")]
8050          VLDRWQ_F))
8051   ]
8052   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8053 {
8054    rtx ops[2];
8055    int regno = REGNO (operands[0]);
8056    ops[0] = gen_rtx_REG (TImode, regno);
8057    ops[1]  = operands[1];
8058    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8059    return "";
8060 }
8061   [(set_attr "length" "8")])
8062
8063 ;;
8064 ;; [vldrwq_z_s vldrwq_z_u]
8065 ;;
8066 (define_insn "mve_vldrwq_z_<supf>v4si"
8067   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8068         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
8069         (match_operand:HI 2 "vpr_register_operand" "Up")]
8070          VLDRWQ))
8071   ]
8072   "TARGET_HAVE_MVE"
8073 {
8074    rtx ops[2];
8075    int regno = REGNO (operands[0]);
8076    ops[0] = gen_rtx_REG (TImode, regno);
8077    ops[1]  = operands[1];
8078    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8079    return "";
8080 }
8081   [(set_attr "length" "8")])
8082
8083 (define_expand "mve_vld1q_f<mode>"
8084   [(match_operand:MVE_0 0 "s_register_operand")
8085    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
8086   ]
8087   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8088 {
8089   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8090   DONE;
8091 })
8092
8093 (define_expand "mve_vld1q_<supf><mode>"
8094   [(match_operand:MVE_2 0 "s_register_operand")
8095    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
8096   ]
8097   "TARGET_HAVE_MVE"
8098 {
8099   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8100   DONE;
8101 })
8102
8103 ;;
8104 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8105 ;;
8106 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8107   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8108         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8109                       (match_operand:SI 2 "immediate_operand" "i")]
8110          VLDRDGBQ))
8111   ]
8112   "TARGET_HAVE_MVE"
8113 {
8114    rtx ops[3];
8115    ops[0] = operands[0];
8116    ops[1] = operands[1];
8117    ops[2] = operands[2];
8118    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8119    return "";
8120 }
8121   [(set_attr "length" "4")])
8122
8123 ;;
8124 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8125 ;;
8126 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8127   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8128         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8129                       (match_operand:SI 2 "immediate_operand" "i")
8130                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8131          VLDRDGBQ))
8132   ]
8133   "TARGET_HAVE_MVE"
8134 {
8135    rtx ops[3];
8136    ops[0] = operands[0];
8137    ops[1] = operands[1];
8138    ops[2] = operands[2];
8139    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8140    return "";
8141 }
8142   [(set_attr "length" "8")])
8143
8144 ;;
8145 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8146 ;;
8147 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8148  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8149        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8150                      (match_operand:V2DI 2 "s_register_operand" "w")]
8151         VLDRDGOQ))
8152  ]
8153  "TARGET_HAVE_MVE"
8154 {
8155   rtx ops[3];
8156   ops[0] = operands[0];
8157   ops[1] = operands[1];
8158   ops[2] = operands[2];
8159   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8160   return "";
8161 }
8162  [(set_attr "length" "4")])
8163
8164 ;;
8165 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8166 ;;
8167 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8168  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8169        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8170                      (match_operand:V2DI 2 "s_register_operand" "w")
8171                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8172         VLDRDGOQ))
8173  ]
8174  "TARGET_HAVE_MVE"
8175 {
8176   rtx ops[3];
8177   ops[0] = operands[0];
8178   ops[1] = operands[1];
8179   ops[2] = operands[2];
8180   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8181   return "";
8182 }
8183  [(set_attr "length" "8")])
8184
8185 ;;
8186 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8187 ;;
8188 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8189   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8190         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8191                       (match_operand:V2DI 2 "s_register_operand" "w")]
8192          VLDRDGSOQ))
8193   ]
8194   "TARGET_HAVE_MVE"
8195 {
8196    rtx ops[3];
8197    ops[0] = operands[0];
8198    ops[1] = operands[1];
8199    ops[2] = operands[2];
8200    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8201    return "";
8202 }
8203   [(set_attr "length" "4")])
8204
8205 ;;
8206 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8207 ;;
8208 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8209   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8210         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8211                       (match_operand:V2DI 2 "s_register_operand" "w")
8212                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8213          VLDRDGSOQ))
8214   ]
8215   "TARGET_HAVE_MVE"
8216 {
8217    rtx ops[3];
8218    ops[0] = operands[0];
8219    ops[1] = operands[1];
8220    ops[2] = operands[2];
8221    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8222    return "";
8223 }
8224   [(set_attr "length" "8")])
8225
8226 ;;
8227 ;; [vldrhq_gather_offset_f]
8228 ;;
8229 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8230   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8231         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8232                       (match_operand:V8HI 2 "s_register_operand" "w")]
8233          VLDRHQGO_F))
8234   ]
8235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8236 {
8237    rtx ops[3];
8238    ops[0] = operands[0];
8239    ops[1] = operands[1];
8240    ops[2] = operands[2];
8241    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8242    return "";
8243 }
8244   [(set_attr "length" "4")])
8245
8246 ;;
8247 ;; [vldrhq_gather_offset_z_f]
8248 ;;
8249 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8250   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8251         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8252                       (match_operand:V8HI 2 "s_register_operand" "w")
8253                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8254          VLDRHQGO_F))
8255   ]
8256   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8257 {
8258    rtx ops[4];
8259    ops[0] = operands[0];
8260    ops[1] = operands[1];
8261    ops[2] = operands[2];
8262    ops[3] = operands[3];
8263    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8264    return "";
8265 }
8266   [(set_attr "length" "8")])
8267
8268 ;;
8269 ;; [vldrhq_gather_shifted_offset_f]
8270 ;;
8271 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8272   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8273         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8274                       (match_operand:V8HI 2 "s_register_operand" "w")]
8275          VLDRHQGSO_F))
8276   ]
8277   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8278 {
8279    rtx ops[3];
8280    ops[0] = operands[0];
8281    ops[1] = operands[1];
8282    ops[2] = operands[2];
8283    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8284    return "";
8285 }
8286   [(set_attr "length" "4")])
8287
8288 ;;
8289 ;; [vldrhq_gather_shifted_offset_z_f]
8290 ;;
8291 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8292   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8293         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8294                       (match_operand:V8HI 2 "s_register_operand" "w")
8295                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8296          VLDRHQGSO_F))
8297   ]
8298   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8299 {
8300    rtx ops[4];
8301    ops[0] = operands[0];
8302    ops[1] = operands[1];
8303    ops[2] = operands[2];
8304    ops[3] = operands[3];
8305    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8306    return "";
8307 }
8308   [(set_attr "length" "8")])
8309
8310 ;;
8311 ;; [vldrwq_gather_base_f]
8312 ;;
8313 (define_insn "mve_vldrwq_gather_base_fv4sf"
8314   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8315         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8316                       (match_operand:SI 2 "immediate_operand" "i")]
8317          VLDRWQGB_F))
8318   ]
8319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8320 {
8321    rtx ops[3];
8322    ops[0] = operands[0];
8323    ops[1] = operands[1];
8324    ops[2] = operands[2];
8325    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8326    return "";
8327 }
8328   [(set_attr "length" "4")])
8329
8330 ;;
8331 ;; [vldrwq_gather_base_z_f]
8332 ;;
8333 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8334   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8335         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8336                       (match_operand:SI 2 "immediate_operand" "i")
8337                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8338          VLDRWQGB_F))
8339   ]
8340   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8341 {
8342    rtx ops[3];
8343    ops[0] = operands[0];
8344    ops[1] = operands[1];
8345    ops[2] = operands[2];
8346    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8347    return "";
8348 }
8349   [(set_attr "length" "8")])
8350
8351 ;;
8352 ;; [vldrwq_gather_offset_f]
8353 ;;
8354 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8355   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8356         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8357                        (match_operand:V4SI 2 "s_register_operand" "w")]
8358          VLDRWQGO_F))
8359   ]
8360   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8361 {
8362    rtx ops[3];
8363    ops[0] = operands[0];
8364    ops[1] = operands[1];
8365    ops[2] = operands[2];
8366    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8367    return "";
8368 }
8369   [(set_attr "length" "4")])
8370
8371 ;;
8372 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8373 ;;
8374 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8375   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8376         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8377                        (match_operand:V4SI 2 "s_register_operand" "w")]
8378          VLDRWGOQ))
8379   ]
8380   "TARGET_HAVE_MVE"
8381 {
8382    rtx ops[3];
8383    ops[0] = operands[0];
8384    ops[1] = operands[1];
8385    ops[2] = operands[2];
8386    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8387    return "";
8388 }
8389   [(set_attr "length" "4")])
8390
8391 ;;
8392 ;; [vldrwq_gather_offset_z_f]
8393 ;;
8394 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8395   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8396         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8397                       (match_operand:V4SI 2 "s_register_operand" "w")
8398                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8399          VLDRWQGO_F))
8400   ]
8401   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8402 {
8403    rtx ops[4];
8404    ops[0] = operands[0];
8405    ops[1] = operands[1];
8406    ops[2] = operands[2];
8407    ops[3] = operands[3];
8408    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8409    return "";
8410 }
8411   [(set_attr "length" "8")])
8412
8413 ;;
8414 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8415 ;;
8416 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8417   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8418         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8419                       (match_operand:V4SI 2 "s_register_operand" "w")
8420                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8421          VLDRWGOQ))
8422   ]
8423   "TARGET_HAVE_MVE"
8424 {
8425    rtx ops[4];
8426    ops[0] = operands[0];
8427    ops[1] = operands[1];
8428    ops[2] = operands[2];
8429    ops[3] = operands[3];
8430    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8431    return "";
8432 }
8433   [(set_attr "length" "8")])
8434
8435 ;;
8436 ;; [vldrwq_gather_shifted_offset_f]
8437 ;;
8438 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8439   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8440         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8441                       (match_operand:V4SI 2 "s_register_operand" "w")]
8442          VLDRWQGSO_F))
8443   ]
8444   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8445 {
8446    rtx ops[3];
8447    ops[0] = operands[0];
8448    ops[1] = operands[1];
8449    ops[2] = operands[2];
8450    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8451    return "";
8452 }
8453   [(set_attr "length" "4")])
8454
8455 ;;
8456 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8457 ;;
8458 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8459   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8460         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8461                       (match_operand:V4SI 2 "s_register_operand" "w")]
8462          VLDRWGSOQ))
8463   ]
8464   "TARGET_HAVE_MVE"
8465 {
8466    rtx ops[3];
8467    ops[0] = operands[0];
8468    ops[1] = operands[1];
8469    ops[2] = operands[2];
8470    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8471    return "";
8472 }
8473   [(set_attr "length" "4")])
8474
8475 ;;
8476 ;; [vldrwq_gather_shifted_offset_z_f]
8477 ;;
8478 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8479   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8480         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8481                       (match_operand:V4SI 2 "s_register_operand" "w")
8482                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8483          VLDRWQGSO_F))
8484   ]
8485   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8486 {
8487    rtx ops[4];
8488    ops[0] = operands[0];
8489    ops[1] = operands[1];
8490    ops[2] = operands[2];
8491    ops[3] = operands[3];
8492    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8493    return "";
8494 }
8495   [(set_attr "length" "8")])
8496
8497 ;;
8498 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8499 ;;
8500 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8501   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8502         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8503                       (match_operand:V4SI 2 "s_register_operand" "w")
8504                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8505          VLDRWGSOQ))
8506   ]
8507   "TARGET_HAVE_MVE"
8508 {
8509    rtx ops[4];
8510    ops[0] = operands[0];
8511    ops[1] = operands[1];
8512    ops[2] = operands[2];
8513    ops[3] = operands[3];
8514    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8515    return "";
8516 }
8517   [(set_attr "length" "8")])
8518
8519 ;;
8520 ;; [vstrhq_f]
8521 ;;
8522 (define_insn "mve_vstrhq_fv8hf"
8523   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8524         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8525          VSTRHQ_F))
8526   ]
8527   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8528 {
8529    rtx ops[2];
8530    int regno = REGNO (operands[1]);
8531    ops[1] = gen_rtx_REG (TImode, regno);
8532    ops[0]  = operands[0];
8533    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8534    return "";
8535 }
8536   [(set_attr "length" "4")])
8537
8538 ;;
8539 ;; [vstrhq_p_f]
8540 ;;
8541 (define_insn "mve_vstrhq_p_fv8hf"
8542   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8543         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8544                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8545          VSTRHQ_F))
8546   ]
8547   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8548 {
8549    rtx ops[2];
8550    int regno = REGNO (operands[1]);
8551    ops[1] = gen_rtx_REG (TImode, regno);
8552    ops[0]  = operands[0];
8553    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8554    return "";
8555 }
8556   [(set_attr "length" "8")])
8557
8558 ;;
8559 ;; [vstrhq_p_s vstrhq_p_u]
8560 ;;
8561 (define_insn "mve_vstrhq_p_<supf><mode>"
8562   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8563         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8564                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8565          VSTRHQ))
8566   ]
8567   "TARGET_HAVE_MVE"
8568 {
8569    rtx ops[2];
8570    int regno = REGNO (operands[1]);
8571    ops[1] = gen_rtx_REG (TImode, regno);
8572    ops[0]  = operands[0];
8573    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8574    return "";
8575 }
8576   [(set_attr "length" "8")])
8577
8578 ;;
8579 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8580 ;;
8581 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8582   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8583    (match_operand:MVE_6 1 "s_register_operand")
8584    (match_operand:MVE_6 2 "s_register_operand")
8585    (match_operand:HI 3 "vpr_register_operand")
8586    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8587   "TARGET_HAVE_MVE"
8588 {
8589   rtx ind = XEXP (operands[0], 0);
8590   gcc_assert (REG_P (ind));
8591   emit_insn (
8592     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8593                                                        operands[2],
8594                                                        operands[3]));
8595   DONE;
8596 })
8597
8598 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8599   [(set (mem:BLK (scratch))
8600         (unspec:BLK
8601           [(match_operand:SI 0 "register_operand" "r")
8602            (match_operand:MVE_6 1 "s_register_operand" "w")
8603            (match_operand:MVE_6 2 "s_register_operand" "w")
8604            (match_operand:HI 3 "vpr_register_operand" "Up")]
8605           VSTRHSOQ))]
8606   "TARGET_HAVE_MVE"
8607   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8608   [(set_attr "length" "8")])
8609
8610 ;;
8611 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8612 ;;
8613 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8614   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8615    (match_operand:MVE_6 1 "s_register_operand")
8616    (match_operand:MVE_6 2 "s_register_operand")
8617    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8618   "TARGET_HAVE_MVE"
8619 {
8620   rtx ind = XEXP (operands[0], 0);
8621   gcc_assert (REG_P (ind));
8622   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8623                                                               operands[2]));
8624   DONE;
8625 })
8626
8627 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8628   [(set (mem:BLK (scratch))
8629         (unspec:BLK
8630           [(match_operand:SI 0 "register_operand" "r")
8631            (match_operand:MVE_6 1 "s_register_operand" "w")
8632            (match_operand:MVE_6 2 "s_register_operand" "w")]
8633           VSTRHSOQ))]
8634   "TARGET_HAVE_MVE"
8635   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8636   [(set_attr "length" "4")])
8637
8638 ;;
8639 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8640 ;;
8641 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8642   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8643    (match_operand:MVE_6 1 "s_register_operand")
8644    (match_operand:MVE_6 2 "s_register_operand")
8645    (match_operand:HI 3 "vpr_register_operand")
8646    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8647   "TARGET_HAVE_MVE"
8648 {
8649   rtx ind = XEXP (operands[0], 0);
8650   gcc_assert (REG_P (ind));
8651   emit_insn (
8652     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8653                                                                operands[2],
8654                                                                operands[3]));
8655   DONE;
8656 })
8657
8658 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8659   [(set (mem:BLK (scratch))
8660         (unspec:BLK
8661           [(match_operand:SI 0 "register_operand" "r")
8662            (match_operand:MVE_6 1 "s_register_operand" "w")
8663            (match_operand:MVE_6 2 "s_register_operand" "w")
8664            (match_operand:HI 3 "vpr_register_operand" "Up")]
8665           VSTRHSSOQ))]
8666   "TARGET_HAVE_MVE"
8667   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8668   [(set_attr "length" "8")])
8669
8670 ;;
8671 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8672 ;;
8673 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8674   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8675    (match_operand:MVE_6 1 "s_register_operand")
8676    (match_operand:MVE_6 2 "s_register_operand")
8677    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8678   "TARGET_HAVE_MVE"
8679 {
8680   rtx ind = XEXP (operands[0], 0);
8681   gcc_assert (REG_P (ind));
8682   emit_insn (
8683     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8684                                                              operands[2]));
8685   DONE;
8686 })
8687
8688 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8689   [(set (mem:BLK (scratch))
8690         (unspec:BLK
8691           [(match_operand:SI 0 "register_operand" "r")
8692            (match_operand:MVE_6 1 "s_register_operand" "w")
8693            (match_operand:MVE_6 2 "s_register_operand" "w")]
8694           VSTRHSSOQ))]
8695   "TARGET_HAVE_MVE"
8696   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8697   [(set_attr "length" "4")])
8698
8699 ;;
8700 ;; [vstrhq_s, vstrhq_u]
8701 ;;
8702 (define_insn "mve_vstrhq_<supf><mode>"
8703   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8704         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8705          VSTRHQ))
8706   ]
8707   "TARGET_HAVE_MVE"
8708 {
8709    rtx ops[2];
8710    int regno = REGNO (operands[1]);
8711    ops[1] = gen_rtx_REG (TImode, regno);
8712    ops[0]  = operands[0];
8713    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8714    return "";
8715 }
8716   [(set_attr "length" "4")])
8717
8718 ;;
8719 ;; [vstrwq_f]
8720 ;;
8721 (define_insn "mve_vstrwq_fv4sf"
8722   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8723         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8724          VSTRWQ_F))
8725   ]
8726   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8727 {
8728    rtx ops[2];
8729    int regno = REGNO (operands[1]);
8730    ops[1] = gen_rtx_REG (TImode, regno);
8731    ops[0]  = operands[0];
8732    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8733    return "";
8734 }
8735   [(set_attr "length" "4")])
8736
8737 ;;
8738 ;; [vstrwq_p_f]
8739 ;;
8740 (define_insn "mve_vstrwq_p_fv4sf"
8741   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8742         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8743                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8744          VSTRWQ_F))
8745   ]
8746   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8747 {
8748    rtx ops[2];
8749    int regno = REGNO (operands[1]);
8750    ops[1] = gen_rtx_REG (TImode, regno);
8751    ops[0]  = operands[0];
8752    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8753    return "";
8754 }
8755   [(set_attr "length" "8")])
8756
8757 ;;
8758 ;; [vstrwq_p_s vstrwq_p_u]
8759 ;;
8760 (define_insn "mve_vstrwq_p_<supf>v4si"
8761   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8762         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8763                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8764          VSTRWQ))
8765   ]
8766   "TARGET_HAVE_MVE"
8767 {
8768    rtx ops[2];
8769    int regno = REGNO (operands[1]);
8770    ops[1] = gen_rtx_REG (TImode, regno);
8771    ops[0]  = operands[0];
8772    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8773    return "";
8774 }
8775   [(set_attr "length" "8")])
8776
8777 ;;
8778 ;; [vstrwq_s vstrwq_u]
8779 ;;
8780 (define_insn "mve_vstrwq_<supf>v4si"
8781   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8782         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8783          VSTRWQ))
8784   ]
8785   "TARGET_HAVE_MVE"
8786 {
8787    rtx ops[2];
8788    int regno = REGNO (operands[1]);
8789    ops[1] = gen_rtx_REG (TImode, regno);
8790    ops[0]  = operands[0];
8791    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8792    return "";
8793 }
8794   [(set_attr "length" "4")])
8795
8796 (define_expand "mve_vst1q_f<mode>"
8797   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8798    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8799   ]
8800   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8801 {
8802   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8803   DONE;
8804 })
8805
8806 (define_expand "mve_vst1q_<supf><mode>"
8807   [(match_operand:MVE_2 0 "mve_memory_operand")
8808    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8809   ]
8810   "TARGET_HAVE_MVE"
8811 {
8812   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8813   DONE;
8814 })
8815
8816 ;;
8817 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8818 ;;
8819 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8820   [(set (mem:BLK (scratch))
8821         (unspec:BLK
8822                 [(match_operand:V2DI 0 "s_register_operand" "w")
8823                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8824                  (match_operand:V2DI 2 "s_register_operand" "w")
8825                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8826          VSTRDSBQ))
8827   ]
8828   "TARGET_HAVE_MVE"
8829 {
8830    rtx ops[3];
8831    ops[0] = operands[0];
8832    ops[1] = operands[1];
8833    ops[2] = operands[2];
8834    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8835    return "";
8836 }
8837   [(set_attr "length" "8")])
8838
8839 ;;
8840 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8841 ;;
8842 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8843   [(set (mem:BLK (scratch))
8844         (unspec:BLK
8845                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8846                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8847                  (match_operand:V2DI 2 "s_register_operand" "w")]
8848          VSTRDSBQ))
8849   ]
8850   "TARGET_HAVE_MVE"
8851 {
8852    rtx ops[3];
8853    ops[0] = operands[0];
8854    ops[1] = operands[1];
8855    ops[2] = operands[2];
8856    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8857    return "";
8858 }
8859   [(set_attr "length" "4")])
8860
8861 ;;
8862 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8863 ;;
8864 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8865   [(match_operand:V2DI 0 "mve_scatter_memory")
8866    (match_operand:V2DI 1 "s_register_operand")
8867    (match_operand:V2DI 2 "s_register_operand")
8868    (match_operand:HI 3 "vpr_register_operand")
8869    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8870   "TARGET_HAVE_MVE"
8871 {
8872   rtx ind = XEXP (operands[0], 0);
8873   gcc_assert (REG_P (ind));
8874   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8875                                                               operands[2],
8876                                                               operands[3]));
8877   DONE;
8878 })
8879
8880 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8881   [(set (mem:BLK (scratch))
8882         (unspec:BLK
8883           [(match_operand:SI 0 "register_operand" "r")
8884            (match_operand:V2DI 1 "s_register_operand" "w")
8885            (match_operand:V2DI 2 "s_register_operand" "w")
8886            (match_operand:HI 3 "vpr_register_operand" "Up")]
8887           VSTRDSOQ))]
8888   "TARGET_HAVE_MVE"
8889   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8890   [(set_attr "length" "8")])
8891
8892 ;;
8893 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8894 ;;
8895 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8896   [(match_operand:V2DI 0 "mve_scatter_memory")
8897    (match_operand:V2DI 1 "s_register_operand")
8898    (match_operand:V2DI 2 "s_register_operand")
8899    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8900   "TARGET_HAVE_MVE"
8901 {
8902   rtx ind = XEXP (operands[0], 0);
8903   gcc_assert (REG_P (ind));
8904   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8905                                                             operands[2]));
8906   DONE;
8907 })
8908
8909 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8910   [(set (mem:BLK (scratch))
8911         (unspec:BLK
8912           [(match_operand:SI 0 "register_operand" "r")
8913            (match_operand:V2DI 1 "s_register_operand" "w")
8914            (match_operand:V2DI 2 "s_register_operand" "w")]
8915           VSTRDSOQ))]
8916   "TARGET_HAVE_MVE"
8917   "vstrd.64\t%q2, [%0, %q1]"
8918   [(set_attr "length" "4")])
8919
8920 ;;
8921 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8922 ;;
8923 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8924   [(match_operand:V2DI 0 "mve_scatter_memory")
8925    (match_operand:V2DI 1 "s_register_operand")
8926    (match_operand:V2DI 2 "s_register_operand")
8927    (match_operand:HI 3 "vpr_register_operand")
8928    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8929   "TARGET_HAVE_MVE"
8930 {
8931   rtx ind = XEXP (operands[0], 0);
8932   gcc_assert (REG_P (ind));
8933   emit_insn (
8934     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8935                                                              operands[2],
8936                                                              operands[3]));
8937   DONE;
8938 })
8939
8940 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8941   [(set (mem:BLK (scratch))
8942         (unspec:BLK
8943           [(match_operand:SI 0 "register_operand" "r")
8944            (match_operand:V2DI 1 "s_register_operand" "w")
8945            (match_operand:V2DI 2 "s_register_operand" "w")
8946            (match_operand:HI 3 "vpr_register_operand" "Up")]
8947           VSTRDSSOQ))]
8948   "TARGET_HAVE_MVE"
8949   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8950   [(set_attr "length" "8")])
8951
8952 ;;
8953 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8954 ;;
8955 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8956   [(match_operand:V2DI 0 "mve_scatter_memory")
8957    (match_operand:V2DI 1 "s_register_operand")
8958    (match_operand:V2DI 2 "s_register_operand")
8959    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8960   "TARGET_HAVE_MVE"
8961 {
8962   rtx ind = XEXP (operands[0], 0);
8963   gcc_assert (REG_P (ind));
8964   emit_insn (
8965     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8966                                                            operands[2]));
8967   DONE;
8968 })
8969
8970 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8971   [(set (mem:BLK (scratch))
8972         (unspec:BLK
8973           [(match_operand:SI 0 "register_operand" "r")
8974            (match_operand:V2DI 1 "s_register_operand" "w")
8975            (match_operand:V2DI 2 "s_register_operand" "w")]
8976           VSTRDSSOQ))]
8977   "TARGET_HAVE_MVE"
8978   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8979   [(set_attr "length" "4")])
8980
8981 ;;
8982 ;; [vstrhq_scatter_offset_f]
8983 ;;
8984 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8985   [(match_operand:V8HI 0 "mve_scatter_memory")
8986    (match_operand:V8HI 1 "s_register_operand")
8987    (match_operand:V8HF 2 "s_register_operand")
8988    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8989   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8990 {
8991   rtx ind = XEXP (operands[0], 0);
8992   gcc_assert (REG_P (ind));
8993   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8994                                                        operands[2]));
8995   DONE;
8996 })
8997
8998 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8999   [(set (mem:BLK (scratch))
9000         (unspec:BLK
9001           [(match_operand:SI 0 "register_operand" "r")
9002            (match_operand:V8HI 1 "s_register_operand" "w")
9003            (match_operand:V8HF 2 "s_register_operand" "w")]
9004           VSTRHQSO_F))]
9005   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9006   "vstrh.16\t%q2, [%0, %q1]"
9007   [(set_attr "length" "4")])
9008
9009 ;;
9010 ;; [vstrhq_scatter_offset_p_f]
9011 ;;
9012 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
9013   [(match_operand:V8HI 0 "mve_scatter_memory")
9014    (match_operand:V8HI 1 "s_register_operand")
9015    (match_operand:V8HF 2 "s_register_operand")
9016    (match_operand:HI 3 "vpr_register_operand")
9017    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
9018   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9019 {
9020   rtx ind = XEXP (operands[0], 0);
9021   gcc_assert (REG_P (ind));
9022   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
9023                                                          operands[2],
9024                                                          operands[3]));
9025   DONE;
9026 })
9027
9028 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
9029   [(set (mem:BLK (scratch))
9030         (unspec:BLK
9031           [(match_operand:SI 0 "register_operand" "r")
9032            (match_operand:V8HI 1 "s_register_operand" "w")
9033            (match_operand:V8HF 2 "s_register_operand" "w")
9034            (match_operand:HI 3 "vpr_register_operand" "Up")]
9035           VSTRHQSO_F))]
9036   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9037   "vpst\;vstrht.16\t%q2, [%0, %q1]"
9038   [(set_attr "length" "8")])
9039
9040 ;;
9041 ;; [vstrhq_scatter_shifted_offset_f]
9042 ;;
9043 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
9044   [(match_operand:V8HI 0 "memory_operand" "=Us")
9045    (match_operand:V8HI 1 "s_register_operand" "w")
9046    (match_operand:V8HF 2 "s_register_operand" "w")
9047    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9048   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9049 {
9050   rtx ind = XEXP (operands[0], 0);
9051   gcc_assert (REG_P (ind));
9052   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
9053                                                                operands[2]));
9054   DONE;
9055 })
9056
9057 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
9058   [(set (mem:BLK (scratch))
9059         (unspec:BLK
9060           [(match_operand:SI 0 "register_operand" "r")
9061            (match_operand:V8HI 1 "s_register_operand" "w")
9062            (match_operand:V8HF 2 "s_register_operand" "w")]
9063           VSTRHQSSO_F))]
9064   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9065   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
9066   [(set_attr "length" "4")])
9067
9068 ;;
9069 ;; [vstrhq_scatter_shifted_offset_p_f]
9070 ;;
9071 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9072   [(match_operand:V8HI 0 "memory_operand" "=Us")
9073    (match_operand:V8HI 1 "s_register_operand" "w")
9074    (match_operand:V8HF 2 "s_register_operand" "w")
9075    (match_operand:HI 3 "vpr_register_operand" "Up")
9076    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9077   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9078 {
9079   rtx ind = XEXP (operands[0], 0);
9080   gcc_assert (REG_P (ind));
9081   emit_insn (
9082     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
9083                                                         operands[2],
9084                                                         operands[3]));
9085   DONE;
9086 })
9087
9088 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
9089   [(set (mem:BLK (scratch))
9090         (unspec:BLK
9091           [(match_operand:SI 0 "register_operand" "r")
9092            (match_operand:V8HI 1 "s_register_operand" "w")
9093            (match_operand:V8HF 2 "s_register_operand" "w")
9094            (match_operand:HI 3 "vpr_register_operand" "Up")]
9095           VSTRHQSSO_F))]
9096   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9097   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
9098   [(set_attr "length" "8")])
9099
9100 ;;
9101 ;; [vstrwq_scatter_base_f]
9102 ;;
9103 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9104   [(set (mem:BLK (scratch))
9105         (unspec:BLK
9106                 [(match_operand:V4SI 0 "s_register_operand" "w")
9107                  (match_operand:SI 1 "immediate_operand" "i")
9108                  (match_operand:V4SF 2 "s_register_operand" "w")]
9109          VSTRWQSB_F))
9110   ]
9111   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9112 {
9113    rtx ops[3];
9114    ops[0] = operands[0];
9115    ops[1] = operands[1];
9116    ops[2] = operands[2];
9117    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9118    return "";
9119 }
9120   [(set_attr "length" "4")])
9121
9122 ;;
9123 ;; [vstrwq_scatter_base_p_f]
9124 ;;
9125 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9126   [(set (mem:BLK (scratch))
9127         (unspec:BLK
9128                 [(match_operand:V4SI 0 "s_register_operand" "w")
9129                  (match_operand:SI 1 "immediate_operand" "i")
9130                  (match_operand:V4SF 2 "s_register_operand" "w")
9131                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9132          VSTRWQSB_F))
9133   ]
9134   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9135 {
9136    rtx ops[3];
9137    ops[0] = operands[0];
9138    ops[1] = operands[1];
9139    ops[2] = operands[2];
9140    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9141    return "";
9142 }
9143   [(set_attr "length" "8")])
9144
9145 ;;
9146 ;; [vstrwq_scatter_offset_f]
9147 ;;
9148 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9149   [(match_operand:V4SI 0 "mve_scatter_memory")
9150    (match_operand:V4SI 1 "s_register_operand")
9151    (match_operand:V4SF 2 "s_register_operand")
9152    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9153   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9154 {
9155   rtx ind = XEXP (operands[0], 0);
9156   gcc_assert (REG_P (ind));
9157   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9158                                                        operands[2]));
9159   DONE;
9160 })
9161
9162 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9163   [(set (mem:BLK (scratch))
9164         (unspec:BLK
9165           [(match_operand:SI 0 "register_operand" "r")
9166            (match_operand:V4SI 1 "s_register_operand" "w")
9167            (match_operand:V4SF 2 "s_register_operand" "w")]
9168           VSTRWQSO_F))]
9169   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9170   "vstrw.32\t%q2, [%0, %q1]"
9171   [(set_attr "length" "4")])
9172
9173 ;;
9174 ;; [vstrwq_scatter_offset_p_f]
9175 ;;
9176 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9177   [(match_operand:V4SI 0 "mve_scatter_memory")
9178    (match_operand:V4SI 1 "s_register_operand")
9179    (match_operand:V4SF 2 "s_register_operand")
9180    (match_operand:HI 3 "vpr_register_operand")
9181    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9182   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9183 {
9184   rtx ind = XEXP (operands[0], 0);
9185   gcc_assert (REG_P (ind));
9186   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9187                                                          operands[2],
9188                                                          operands[3]));
9189   DONE;
9190 })
9191
9192 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9193   [(set (mem:BLK (scratch))
9194         (unspec:BLK
9195           [(match_operand:SI 0 "register_operand" "r")
9196            (match_operand:V4SI 1 "s_register_operand" "w")
9197            (match_operand:V4SF 2 "s_register_operand" "w")
9198            (match_operand:HI 3 "vpr_register_operand" "Up")]
9199           VSTRWQSO_F))]
9200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9201   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9202   [(set_attr "length" "8")])
9203
9204 ;;
9205 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9206 ;;
9207 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9208   [(match_operand:V4SI 0 "mve_scatter_memory")
9209    (match_operand:V4SI 1 "s_register_operand")
9210    (match_operand:V4SI 2 "s_register_operand")
9211    (match_operand:HI 3 "vpr_register_operand")
9212    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9213   "TARGET_HAVE_MVE"
9214 {
9215   rtx ind = XEXP (operands[0], 0);
9216   gcc_assert (REG_P (ind));
9217   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9218                                                               operands[2],
9219                                                               operands[3]));
9220   DONE;
9221 })
9222
9223 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9224   [(set (mem:BLK (scratch))
9225         (unspec:BLK
9226           [(match_operand:SI 0 "register_operand" "r")
9227            (match_operand:V4SI 1 "s_register_operand" "w")
9228            (match_operand:V4SI 2 "s_register_operand" "w")
9229            (match_operand:HI 3 "vpr_register_operand" "Up")]
9230           VSTRWSOQ))]
9231   "TARGET_HAVE_MVE"
9232   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9233   [(set_attr "length" "8")])
9234
9235 ;;
9236 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9237 ;;
9238 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9239   [(match_operand:V4SI 0 "mve_scatter_memory")
9240    (match_operand:V4SI 1 "s_register_operand")
9241    (match_operand:V4SI 2 "s_register_operand")
9242    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9243   "TARGET_HAVE_MVE"
9244 {
9245   rtx ind = XEXP (operands[0], 0);
9246   gcc_assert (REG_P (ind));
9247   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9248                                                             operands[2]));
9249   DONE;
9250 })
9251
9252 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9253   [(set (mem:BLK (scratch))
9254         (unspec:BLK
9255           [(match_operand:SI 0 "register_operand" "r")
9256            (match_operand:V4SI 1 "s_register_operand" "w")
9257            (match_operand:V4SI 2 "s_register_operand" "w")]
9258           VSTRWSOQ))]
9259   "TARGET_HAVE_MVE"
9260   "vstrw.32\t%q2, [%0, %q1]"
9261   [(set_attr "length" "4")])
9262
9263 ;;
9264 ;; [vstrwq_scatter_shifted_offset_f]
9265 ;;
9266 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9267   [(match_operand:V4SI 0 "mve_scatter_memory")
9268    (match_operand:V4SI 1 "s_register_operand")
9269    (match_operand:V4SF 2 "s_register_operand")
9270    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9271   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9272 {
9273   rtx ind = XEXP (operands[0], 0);
9274   gcc_assert (REG_P (ind));
9275   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9276                                                                operands[2]));
9277   DONE;
9278 })
9279
9280 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9281   [(set (mem:BLK (scratch))
9282         (unspec:BLK
9283           [(match_operand:SI 0 "register_operand" "r")
9284            (match_operand:V4SI 1 "s_register_operand" "w")
9285            (match_operand:V4SF 2 "s_register_operand" "w")]
9286          VSTRWQSSO_F))]
9287   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9288   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9289   [(set_attr "length" "8")])
9290
9291 ;;
9292 ;; [vstrwq_scatter_shifted_offset_p_f]
9293 ;;
9294 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9295   [(match_operand:V4SI 0 "mve_scatter_memory")
9296    (match_operand:V4SI 1 "s_register_operand")
9297    (match_operand:V4SF 2 "s_register_operand")
9298    (match_operand:HI 3 "vpr_register_operand")
9299    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9300   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9301 {
9302   rtx ind = XEXP (operands[0], 0);
9303   gcc_assert (REG_P (ind));
9304   emit_insn (
9305     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9306                                                         operands[2],
9307                                                         operands[3]));
9308   DONE;
9309 })
9310
9311 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9312   [(set (mem:BLK (scratch))
9313         (unspec:BLK
9314           [(match_operand:SI 0 "register_operand" "r")
9315            (match_operand:V4SI 1 "s_register_operand" "w")
9316            (match_operand:V4SF 2 "s_register_operand" "w")
9317            (match_operand:HI 3 "vpr_register_operand" "Up")]
9318           VSTRWQSSO_F))]
9319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9320   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9321   [(set_attr "length" "8")])
9322
9323 ;;
9324 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9325 ;;
9326 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9327   [(match_operand:V4SI 0 "mve_scatter_memory")
9328    (match_operand:V4SI 1 "s_register_operand")
9329    (match_operand:V4SI 2 "s_register_operand")
9330    (match_operand:HI 3 "vpr_register_operand")
9331    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9332   "TARGET_HAVE_MVE"
9333 {
9334   rtx ind = XEXP (operands[0], 0);
9335   gcc_assert (REG_P (ind));
9336   emit_insn (
9337     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9338                                                              operands[2],
9339                                                              operands[3]));
9340   DONE;
9341 })
9342
9343 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9344   [(set (mem:BLK (scratch))
9345         (unspec:BLK
9346           [(match_operand:SI 0 "register_operand" "r")
9347            (match_operand:V4SI 1 "s_register_operand" "w")
9348            (match_operand:V4SI 2 "s_register_operand" "w")
9349            (match_operand:HI 3 "vpr_register_operand" "Up")]
9350           VSTRWSSOQ))]
9351   "TARGET_HAVE_MVE"
9352   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9353   [(set_attr "length" "8")])
9354
9355 ;;
9356 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9357 ;;
9358 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9359   [(match_operand:V4SI 0 "mve_scatter_memory")
9360    (match_operand:V4SI 1 "s_register_operand")
9361    (match_operand:V4SI 2 "s_register_operand")
9362    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9363   "TARGET_HAVE_MVE"
9364 {
9365   rtx ind = XEXP (operands[0], 0);
9366   gcc_assert (REG_P (ind));
9367   emit_insn (
9368     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9369                                                            operands[2]));
9370   DONE;
9371 })
9372
9373 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9374   [(set (mem:BLK (scratch))
9375         (unspec:BLK
9376           [(match_operand:SI 0 "register_operand" "r")
9377            (match_operand:V4SI 1 "s_register_operand" "w")
9378            (match_operand:V4SI 2 "s_register_operand" "w")]
9379           VSTRWSSOQ))]
9380   "TARGET_HAVE_MVE"
9381   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9382   [(set_attr "length" "4")])
9383
9384 ;;
9385 ;; [vaddq_s, vaddq_u])
9386 ;;
9387 (define_insn "mve_vaddq<mode>"
9388   [
9389    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9390         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9391                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9392   ]
9393   "TARGET_HAVE_MVE"
9394   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9395   [(set_attr "type" "mve_move")
9396 ])
9397
9398 ;;
9399 ;; [vaddq_f])
9400 ;;
9401 (define_insn "mve_vaddq_f<mode>"
9402   [
9403    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9404         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9405                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9406   ]
9407   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9408   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9409   [(set_attr "type" "mve_move")
9410 ])
9411
9412 ;;
9413 ;; [vidupq_n_u])
9414 ;;
9415 (define_expand "mve_vidupq_n_u<mode>"
9416  [(match_operand:MVE_2 0 "s_register_operand")
9417   (match_operand:SI 1 "s_register_operand")
9418   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9419  "TARGET_HAVE_MVE"
9420 {
9421   rtx temp = gen_reg_rtx (SImode);
9422   emit_move_insn (temp, operands[1]);
9423   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9424   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9425                                           operands[2], inc));
9426   DONE;
9427 })
9428
9429 ;;
9430 ;; [vidupq_u_insn])
9431 ;;
9432 (define_insn "mve_vidupq_u<mode>_insn"
9433  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9434        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9435                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9436          VIDUPQ))
9437   (set (match_operand:SI 1 "s_register_operand" "=Te")
9438        (plus:SI (match_dup 2)
9439                 (match_operand:SI 4 "immediate_operand" "i")))]
9440  "TARGET_HAVE_MVE"
9441  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9442
9443 ;;
9444 ;; [vidupq_m_n_u])
9445 ;;
9446 (define_expand "mve_vidupq_m_n_u<mode>"
9447   [(match_operand:MVE_2 0 "s_register_operand")
9448    (match_operand:MVE_2 1 "s_register_operand")
9449    (match_operand:SI 2 "s_register_operand")
9450    (match_operand:SI 3 "mve_imm_selective_upto_8")
9451    (match_operand:HI 4 "vpr_register_operand")]
9452   "TARGET_HAVE_MVE"
9453 {
9454   rtx temp = gen_reg_rtx (SImode);
9455   emit_move_insn (temp, operands[2]);
9456   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9457   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9458                                              operands[2], operands[3],
9459                                              operands[4], inc));
9460   DONE;
9461 })
9462
9463 ;;
9464 ;; [vidupq_m_wb_u_insn])
9465 ;;
9466 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9467  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9468        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9469                       (match_operand:SI 3 "s_register_operand" "2")
9470                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9471                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9472         VIDUPQ_M))
9473   (set (match_operand:SI 2 "s_register_operand" "=Te")
9474        (plus:SI (match_dup 3)
9475                 (match_operand:SI 6 "immediate_operand" "i")))]
9476  "TARGET_HAVE_MVE"
9477  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9478  [(set_attr "length""8")])
9479
9480 ;;
9481 ;; [vddupq_n_u])
9482 ;;
9483 (define_expand "mve_vddupq_n_u<mode>"
9484  [(match_operand:MVE_2 0 "s_register_operand")
9485   (match_operand:SI 1 "s_register_operand")
9486   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9487  "TARGET_HAVE_MVE"
9488 {
9489   rtx temp = gen_reg_rtx (SImode);
9490   emit_move_insn (temp, operands[1]);
9491   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9492   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9493                                           operands[2], inc));
9494   DONE;
9495 })
9496
9497 ;;
9498 ;; [vddupq_u_insn])
9499 ;;
9500 (define_insn "mve_vddupq_u<mode>_insn"
9501  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9502        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9503                       (match_operand:SI 3 "immediate_operand" "i")]
9504         VDDUPQ))
9505   (set (match_operand:SI 1 "s_register_operand" "=Te")
9506        (minus:SI (match_dup 2)
9507                  (match_operand:SI 4 "immediate_operand" "i")))]
9508  "TARGET_HAVE_MVE"
9509  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9510
9511 ;;
9512 ;; [vddupq_m_n_u])
9513 ;;
9514 (define_expand "mve_vddupq_m_n_u<mode>"
9515   [(match_operand:MVE_2 0 "s_register_operand")
9516    (match_operand:MVE_2 1 "s_register_operand")
9517    (match_operand:SI 2 "s_register_operand")
9518    (match_operand:SI 3 "mve_imm_selective_upto_8")
9519    (match_operand:HI 4 "vpr_register_operand")]
9520   "TARGET_HAVE_MVE"
9521 {
9522   rtx temp = gen_reg_rtx (SImode);
9523   emit_move_insn (temp, operands[2]);
9524   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9525   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9526                                              operands[2], operands[3],
9527                                              operands[4], inc));
9528   DONE;
9529 })
9530
9531 ;;
9532 ;; [vddupq_m_wb_u_insn])
9533 ;;
9534 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9535  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9536        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9537                       (match_operand:SI 3 "s_register_operand" "2")
9538                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9539                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9540         VDDUPQ_M))
9541   (set (match_operand:SI 2 "s_register_operand" "=Te")
9542        (minus:SI (match_dup 3)
9543                  (match_operand:SI 6 "immediate_operand" "i")))]
9544  "TARGET_HAVE_MVE"
9545  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9546  [(set_attr "length""8")])
9547
9548 ;;
9549 ;; [vdwdupq_n_u])
9550 ;;
9551 (define_expand "mve_vdwdupq_n_u<mode>"
9552  [(match_operand:MVE_2 0 "s_register_operand")
9553   (match_operand:SI 1 "s_register_operand")
9554   (match_operand:DI 2 "s_register_operand")
9555   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9556  "TARGET_HAVE_MVE"
9557 {
9558   rtx ignore_wb = gen_reg_rtx (SImode);
9559   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9560                                               operands[1], operands[2],
9561                                               operands[3]));
9562   DONE;
9563 })
9564
9565 ;;
9566 ;; [vdwdupq_wb_u])
9567 ;;
9568 (define_expand "mve_vdwdupq_wb_u<mode>"
9569  [(match_operand:SI 0 "s_register_operand")
9570   (match_operand:SI 1 "s_register_operand")
9571   (match_operand:DI 2 "s_register_operand")
9572   (match_operand:SI 3 "mve_imm_selective_upto_8")
9573   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9574  "TARGET_HAVE_MVE"
9575 {
9576   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9577   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9578                                               operands[1], operands[2],
9579                                               operands[3]));
9580   DONE;
9581 })
9582
9583 ;;
9584 ;; [vdwdupq_wb_u_insn])
9585 ;;
9586 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9587   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9588         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9589                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9590                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9591          VDWDUPQ))
9592    (set (match_operand:SI 1 "s_register_operand" "=Te")
9593         (unspec:SI [(match_dup 2)
9594                     (subreg:SI (match_dup 3) 4)
9595                     (match_dup 4)]
9596          VDWDUPQ))]
9597   "TARGET_HAVE_MVE"
9598   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9599 )
9600
9601 ;;
9602 ;; [vdwdupq_m_n_u])
9603 ;;
9604 (define_expand "mve_vdwdupq_m_n_u<mode>"
9605  [(match_operand:MVE_2 0 "s_register_operand")
9606   (match_operand:MVE_2 1 "s_register_operand")
9607   (match_operand:SI 2 "s_register_operand")
9608   (match_operand:DI 3 "s_register_operand")
9609   (match_operand:SI 4 "mve_imm_selective_upto_8")
9610   (match_operand:HI 5 "vpr_register_operand")]
9611  "TARGET_HAVE_MVE"
9612 {
9613   rtx ignore_wb = gen_reg_rtx (SImode);
9614   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9615                                                 operands[1], operands[2],
9616                                                 operands[3], operands[4],
9617                                                 operands[5]));
9618   DONE;
9619 })
9620
9621 ;;
9622 ;; [vdwdupq_m_wb_u])
9623 ;;
9624 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9625  [(match_operand:SI 0 "s_register_operand")
9626   (match_operand:MVE_2 1 "s_register_operand")
9627   (match_operand:SI 2 "s_register_operand")
9628   (match_operand:DI 3 "s_register_operand")
9629   (match_operand:SI 4 "mve_imm_selective_upto_8")
9630   (match_operand:HI 5 "vpr_register_operand")]
9631  "TARGET_HAVE_MVE"
9632 {
9633   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9634   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9635                                                 operands[1], operands[2],
9636                                                 operands[3], operands[4],
9637                                                 operands[5]));
9638   DONE;
9639 })
9640
9641 ;;
9642 ;; [vdwdupq_m_wb_u_insn])
9643 ;;
9644 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9645   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9646         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9647                        (match_operand:SI 3 "s_register_operand" "1")
9648                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9649                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9650                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9651          VDWDUPQ_M))
9652    (set (match_operand:SI 1 "s_register_operand" "=Te")
9653         (unspec:SI [(match_dup 2)
9654                     (match_dup 3)
9655                     (subreg:SI (match_dup 4) 4)
9656                     (match_dup 5)
9657                     (match_dup 6)]
9658          VDWDUPQ_M))
9659   ]
9660   "TARGET_HAVE_MVE"
9661   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9662   [(set_attr "type" "mve_move")
9663    (set_attr "length""8")])
9664
9665 ;;
9666 ;; [viwdupq_n_u])
9667 ;;
9668 (define_expand "mve_viwdupq_n_u<mode>"
9669  [(match_operand:MVE_2 0 "s_register_operand")
9670   (match_operand:SI 1 "s_register_operand")
9671   (match_operand:DI 2 "s_register_operand")
9672   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9673  "TARGET_HAVE_MVE"
9674 {
9675   rtx ignore_wb = gen_reg_rtx (SImode);
9676   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9677                                               operands[1], operands[2],
9678                                               operands[3]));
9679   DONE;
9680 })
9681
9682 ;;
9683 ;; [viwdupq_wb_u])
9684 ;;
9685 (define_expand "mve_viwdupq_wb_u<mode>"
9686  [(match_operand:SI 0 "s_register_operand")
9687   (match_operand:SI 1 "s_register_operand")
9688   (match_operand:DI 2 "s_register_operand")
9689   (match_operand:SI 3 "mve_imm_selective_upto_8")
9690   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9691  "TARGET_HAVE_MVE"
9692 {
9693   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9694   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9695                                               operands[1], operands[2],
9696                                               operands[3]));
9697   DONE;
9698 })
9699
9700 ;;
9701 ;; [viwdupq_wb_u_insn])
9702 ;;
9703 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9704   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9705         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9706                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9707                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9708          VIWDUPQ))
9709    (set (match_operand:SI 1 "s_register_operand" "=Te")
9710         (unspec:SI [(match_dup 2)
9711                     (subreg:SI (match_dup 3) 4)
9712                     (match_dup 4)]
9713          VIWDUPQ))]
9714   "TARGET_HAVE_MVE"
9715   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9716 )
9717
9718 ;;
9719 ;; [viwdupq_m_n_u])
9720 ;;
9721 (define_expand "mve_viwdupq_m_n_u<mode>"
9722  [(match_operand:MVE_2 0 "s_register_operand")
9723   (match_operand:MVE_2 1 "s_register_operand")
9724   (match_operand:SI 2 "s_register_operand")
9725   (match_operand:DI 3 "s_register_operand")
9726   (match_operand:SI 4 "mve_imm_selective_upto_8")
9727   (match_operand:HI 5 "vpr_register_operand")]
9728  "TARGET_HAVE_MVE"
9729 {
9730   rtx ignore_wb = gen_reg_rtx (SImode);
9731   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9732                                                 operands[1], operands[2],
9733                                                 operands[3], operands[4],
9734                                                 operands[5]));
9735   DONE;
9736 })
9737
9738 ;;
9739 ;; [viwdupq_m_wb_u])
9740 ;;
9741 (define_expand "mve_viwdupq_m_wb_u<mode>"
9742  [(match_operand:SI 0 "s_register_operand")
9743   (match_operand:MVE_2 1 "s_register_operand")
9744   (match_operand:SI 2 "s_register_operand")
9745   (match_operand:DI 3 "s_register_operand")
9746   (match_operand:SI 4 "mve_imm_selective_upto_8")
9747   (match_operand:HI 5 "vpr_register_operand")]
9748  "TARGET_HAVE_MVE"
9749 {
9750   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9751   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9752                                                 operands[1], operands[2],
9753                                                 operands[3], operands[4],
9754                                                 operands[5]));
9755   DONE;
9756 })
9757
9758 ;;
9759 ;; [viwdupq_m_wb_u_insn])
9760 ;;
9761 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9762   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9763         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9764                        (match_operand:SI 3 "s_register_operand" "1")
9765                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9766                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9767                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9768          VIWDUPQ_M))
9769    (set (match_operand:SI 1 "s_register_operand" "=Te")
9770         (unspec:SI [(match_dup 2)
9771                     (match_dup 3)
9772                     (subreg:SI (match_dup 4) 4)
9773                     (match_dup 5)
9774                     (match_dup 6)]
9775          VIWDUPQ_M))
9776   ]
9777   "TARGET_HAVE_MVE"
9778   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9779   [(set_attr "type" "mve_move")
9780    (set_attr "length""8")])
9781
9782 ;;
9783 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9784 ;;
9785 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9786   [(set (mem:BLK (scratch))
9787         (unspec:BLK
9788                 [(match_operand:V4SI 1 "s_register_operand" "0")
9789                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9790                  (match_operand:V4SI 3 "s_register_operand" "w")]
9791          VSTRWSBWBQ))
9792    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9793         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9794          VSTRWSBWBQ))
9795   ]
9796   "TARGET_HAVE_MVE"
9797 {
9798    rtx ops[3];
9799    ops[0] = operands[1];
9800    ops[1] = operands[2];
9801    ops[2] = operands[3];
9802    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9803    return "";
9804 }
9805   [(set_attr "length" "4")])
9806
9807 ;;
9808 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9809 ;;
9810 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9811  [(set (mem:BLK (scratch))
9812        (unspec:BLK
9813                 [(match_operand:V4SI 1 "s_register_operand" "0")
9814                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9815                  (match_operand:V4SI 3 "s_register_operand" "w")
9816                  (match_operand:HI 4 "vpr_register_operand")]
9817         VSTRWSBWBQ))
9818    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9819         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9820          VSTRWSBWBQ))
9821   ]
9822   "TARGET_HAVE_MVE"
9823 {
9824    rtx ops[3];
9825    ops[0] = operands[1];
9826    ops[1] = operands[2];
9827    ops[2] = operands[3];
9828    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9829    return "";
9830 }
9831   [(set_attr "length" "8")])
9832
9833 ;;
9834 ;; [vstrwq_scatter_base_wb_f]
9835 ;;
9836 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9837  [(set (mem:BLK (scratch))
9838        (unspec:BLK
9839                 [(match_operand:V4SI 1 "s_register_operand" "0")
9840                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9841                  (match_operand:V4SF 3 "s_register_operand" "w")]
9842          VSTRWQSBWB_F))
9843    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9844         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9845          VSTRWQSBWB_F))
9846   ]
9847   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9848 {
9849    rtx ops[3];
9850    ops[0] = operands[1];
9851    ops[1] = operands[2];
9852    ops[2] = operands[3];
9853    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9854    return "";
9855 }
9856   [(set_attr "length" "4")])
9857
9858 ;;
9859 ;; [vstrwq_scatter_base_wb_p_f]
9860 ;;
9861 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9862  [(set (mem:BLK (scratch))
9863        (unspec:BLK
9864                 [(match_operand:V4SI 1 "s_register_operand" "0")
9865                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9866                  (match_operand:V4SF 3 "s_register_operand" "w")
9867                  (match_operand:HI 4 "vpr_register_operand")]
9868         VSTRWQSBWB_F))
9869    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9870         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9871          VSTRWQSBWB_F))
9872   ]
9873   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9874 {
9875    rtx ops[3];
9876    ops[0] = operands[1];
9877    ops[1] = operands[2];
9878    ops[2] = operands[3];
9879    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9880    return "";
9881 }
9882   [(set_attr "length" "8")])
9883
9884 ;;
9885 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9886 ;;
9887 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9888   [(set (mem:BLK (scratch))
9889         (unspec:BLK
9890                 [(match_operand:V2DI 1 "s_register_operand" "0")
9891                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9892                  (match_operand:V2DI 3 "s_register_operand" "w")]
9893          VSTRDSBWBQ))
9894    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9895         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9896          VSTRDSBWBQ))
9897   ]
9898   "TARGET_HAVE_MVE"
9899 {
9900    rtx ops[3];
9901    ops[0] = operands[1];
9902    ops[1] = operands[2];
9903    ops[2] = operands[3];
9904    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9905    return "";
9906 }
9907   [(set_attr "length" "4")])
9908
9909 ;;
9910 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9911 ;;
9912 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9913   [(set (mem:BLK (scratch))
9914         (unspec:BLK
9915                 [(match_operand:V2DI 1 "s_register_operand" "0")
9916                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9917                  (match_operand:V2DI 3 "s_register_operand" "w")
9918                  (match_operand:HI 4 "vpr_register_operand")]
9919          VSTRDSBWBQ))
9920    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9921         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9922          VSTRDSBWBQ))
9923   ]
9924   "TARGET_HAVE_MVE"
9925 {
9926    rtx ops[3];
9927    ops[0] = operands[1];
9928    ops[1] = operands[2];
9929    ops[2] = operands[3];
9930    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9931    return "";
9932 }
9933   [(set_attr "length" "8")])
9934
9935 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9936   [(match_operand:V4SI 0 "s_register_operand")
9937    (match_operand:V4SI 1 "s_register_operand")
9938    (match_operand:SI 2 "mve_vldrd_immediate")
9939    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9940   "TARGET_HAVE_MVE"
9941 {
9942   rtx ignore_result = gen_reg_rtx (V4SImode);
9943   emit_insn (
9944   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9945                                                  operands[1], operands[2]));
9946   DONE;
9947 })
9948
9949 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9950   [(match_operand:V4SI 0 "s_register_operand")
9951    (match_operand:V4SI 1 "s_register_operand")
9952    (match_operand:SI 2 "mve_vldrd_immediate")
9953    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9954   "TARGET_HAVE_MVE"
9955 {
9956   rtx ignore_wb = gen_reg_rtx (V4SImode);
9957   emit_insn (
9958   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9959                                                  operands[1], operands[2]));
9960   DONE;
9961 })
9962
9963 ;;
9964 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9965 ;;
9966 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9967   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9968         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9969                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9970                       (mem:BLK (scratch))]
9971          VLDRWGBWBQ))
9972    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9973         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9974          VLDRWGBWBQ))
9975   ]
9976   "TARGET_HAVE_MVE"
9977 {
9978    rtx ops[3];
9979    ops[0] = operands[0];
9980    ops[1] = operands[2];
9981    ops[2] = operands[3];
9982    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9983    return "";
9984 }
9985   [(set_attr "length" "4")])
9986
9987 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9988   [(match_operand:V4SI 0 "s_register_operand")
9989    (match_operand:V4SI 1 "s_register_operand")
9990    (match_operand:SI 2 "mve_vldrd_immediate")
9991    (match_operand:HI 3 "vpr_register_operand")
9992    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9993   "TARGET_HAVE_MVE"
9994 {
9995   rtx ignore_result = gen_reg_rtx (V4SImode);
9996   emit_insn (
9997   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9998                                                    operands[1], operands[2],
9999                                                    operands[3]));
10000   DONE;
10001 })
10002 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
10003   [(match_operand:V4SI 0 "s_register_operand")
10004    (match_operand:V4SI 1 "s_register_operand")
10005    (match_operand:SI 2 "mve_vldrd_immediate")
10006    (match_operand:HI 3 "vpr_register_operand")
10007    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10008   "TARGET_HAVE_MVE"
10009 {
10010   rtx ignore_wb = gen_reg_rtx (V4SImode);
10011   emit_insn (
10012   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10013                                                    operands[1], operands[2],
10014                                                    operands[3]));
10015   DONE;
10016 })
10017
10018 ;;
10019 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10020 ;;
10021 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10022   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10023         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10024                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10025                       (match_operand:HI 4 "vpr_register_operand" "Up")
10026                       (mem:BLK (scratch))]
10027          VLDRWGBWBQ))
10028    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10029         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10030          VLDRWGBWBQ))
10031   ]
10032   "TARGET_HAVE_MVE"
10033 {
10034    rtx ops[3];
10035    ops[0] = operands[0];
10036    ops[1] = operands[2];
10037    ops[2] = operands[3];
10038    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10039    return "";
10040 }
10041   [(set_attr "length" "8")])
10042
10043 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10044   [(match_operand:V4SI 0 "s_register_operand")
10045    (match_operand:V4SI 1 "s_register_operand")
10046    (match_operand:SI 2 "mve_vldrd_immediate")
10047    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10048   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10049 {
10050   rtx ignore_result = gen_reg_rtx (V4SFmode);
10051   emit_insn (
10052   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10053                                             operands[1], operands[2]));
10054   DONE;
10055 })
10056
10057 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10058   [(match_operand:V4SF 0 "s_register_operand")
10059    (match_operand:V4SI 1 "s_register_operand")
10060    (match_operand:SI 2 "mve_vldrd_immediate")
10061    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10062   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10063 {
10064   rtx ignore_wb = gen_reg_rtx (V4SImode);
10065   emit_insn (
10066   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10067                                             operands[1], operands[2]));
10068   DONE;
10069 })
10070
10071 ;;
10072 ;; [vldrwq_gather_base_wb_f]
10073 ;;
10074 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10075   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10076         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10077                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10078                       (mem:BLK (scratch))]
10079          VLDRWQGBWB_F))
10080    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10081         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10082          VLDRWQGBWB_F))
10083   ]
10084   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10085 {
10086    rtx ops[3];
10087    ops[0] = operands[0];
10088    ops[1] = operands[2];
10089    ops[2] = operands[3];
10090    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10091    return "";
10092 }
10093   [(set_attr "length" "4")])
10094
10095 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10096   [(match_operand:V4SI 0 "s_register_operand")
10097    (match_operand:V4SI 1 "s_register_operand")
10098    (match_operand:SI 2 "mve_vldrd_immediate")
10099    (match_operand:HI 3 "vpr_register_operand")
10100    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10101   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10102 {
10103   rtx ignore_result = gen_reg_rtx (V4SFmode);
10104   emit_insn (
10105   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10106                                               operands[1], operands[2],
10107                                               operands[3]));
10108   DONE;
10109 })
10110
10111 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10112   [(match_operand:V4SF 0 "s_register_operand")
10113    (match_operand:V4SI 1 "s_register_operand")
10114    (match_operand:SI 2 "mve_vldrd_immediate")
10115    (match_operand:HI 3 "vpr_register_operand")
10116    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10117   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10118 {
10119   rtx ignore_wb = gen_reg_rtx (V4SImode);
10120   emit_insn (
10121   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10122                                               operands[1], operands[2],
10123                                               operands[3]));
10124   DONE;
10125 })
10126
10127 ;;
10128 ;; [vldrwq_gather_base_wb_z_f]
10129 ;;
10130 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10131   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10132         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10133                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10134                       (match_operand:HI 4 "vpr_register_operand" "Up")
10135                       (mem:BLK (scratch))]
10136          VLDRWQGBWB_F))
10137    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10138         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10139          VLDRWQGBWB_F))
10140   ]
10141   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10142 {
10143    rtx ops[3];
10144    ops[0] = operands[0];
10145    ops[1] = operands[2];
10146    ops[2] = operands[3];
10147    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10148    return "";
10149 }
10150   [(set_attr "length" "8")])
10151
10152 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10153   [(match_operand:V2DI 0 "s_register_operand")
10154    (match_operand:V2DI 1 "s_register_operand")
10155    (match_operand:SI 2 "mve_vldrd_immediate")
10156    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10157   "TARGET_HAVE_MVE"
10158 {
10159   rtx ignore_result = gen_reg_rtx (V2DImode);
10160   emit_insn (
10161   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10162                                                  operands[1], operands[2]));
10163   DONE;
10164 })
10165
10166 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10167   [(match_operand:V2DI 0 "s_register_operand")
10168    (match_operand:V2DI 1 "s_register_operand")
10169    (match_operand:SI 2 "mve_vldrd_immediate")
10170    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10171   "TARGET_HAVE_MVE"
10172 {
10173   rtx ignore_wb = gen_reg_rtx (V2DImode);
10174   emit_insn (
10175   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10176                                                  operands[1], operands[2]));
10177   DONE;
10178 })
10179
10180
10181 ;;
10182 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10183 ;;
10184 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10185   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10186         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10187                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10188                       (mem:BLK (scratch))]
10189          VLDRDGBWBQ))
10190    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10191         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10192          VLDRDGBWBQ))
10193   ]
10194   "TARGET_HAVE_MVE"
10195 {
10196    rtx ops[3];
10197    ops[0] = operands[0];
10198    ops[1] = operands[2];
10199    ops[2] = operands[3];
10200    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10201    return "";
10202 }
10203   [(set_attr "length" "4")])
10204
10205 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10206   [(match_operand:V2DI 0 "s_register_operand")
10207    (match_operand:V2DI 1 "s_register_operand")
10208    (match_operand:SI 2 "mve_vldrd_immediate")
10209    (match_operand:HI 3 "vpr_register_operand")
10210    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10211   "TARGET_HAVE_MVE"
10212 {
10213   rtx ignore_result = gen_reg_rtx (V2DImode);
10214   emit_insn (
10215   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10216                                                    operands[1], operands[2],
10217                                                    operands[3]));
10218   DONE;
10219 })
10220
10221 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10222   [(match_operand:V2DI 0 "s_register_operand")
10223    (match_operand:V2DI 1 "s_register_operand")
10224    (match_operand:SI 2 "mve_vldrd_immediate")
10225    (match_operand:HI 3 "vpr_register_operand")
10226    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10227   "TARGET_HAVE_MVE"
10228 {
10229   rtx ignore_wb = gen_reg_rtx (V2DImode);
10230   emit_insn (
10231   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10232                                                    operands[1], operands[2],
10233                                                    operands[3]));
10234   DONE;
10235 })
10236
10237 (define_insn "get_fpscr_nzcvqc"
10238  [(set (match_operand:SI 0 "register_operand" "=r")
10239    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10240  "TARGET_HAVE_MVE"
10241  "vmrs\\t%0, FPSCR_nzcvqc"
10242   [(set_attr "type" "mve_move")])
10243
10244 (define_insn "set_fpscr_nzcvqc"
10245  [(set (reg:SI VFPCC_REGNUM)
10246    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10247     VUNSPEC_SET_FPSCR_NZCVQC))]
10248  "TARGET_HAVE_MVE"
10249  "vmsr\\tFPSCR_nzcvqc, %0"
10250   [(set_attr "type" "mve_move")])
10251
10252 ;;
10253 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10254 ;;
10255 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10256   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10257         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10258                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10259                       (match_operand:HI 4 "vpr_register_operand" "Up")
10260                       (mem:BLK (scratch))]
10261          VLDRDGBWBQ))
10262    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10263         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10264          VLDRDGBWBQ))
10265   ]
10266   "TARGET_HAVE_MVE"
10267 {
10268    rtx ops[3];
10269    ops[0] = operands[0];
10270    ops[1] = operands[2];
10271    ops[2] = operands[3];
10272    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10273    return "";
10274 }
10275   [(set_attr "length" "8")])
10276 ;;
10277 ;; [vadciq_m_s, vadciq_m_u])
10278 ;;
10279 (define_insn "mve_vadciq_m_<supf>v4si"
10280   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10281         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10282                       (match_operand:V4SI 2 "s_register_operand" "w")
10283                       (match_operand:V4SI 3 "s_register_operand" "w")
10284                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10285          VADCIQ_M))
10286    (set (reg:SI VFPCC_REGNUM)
10287         (unspec:SI [(const_int 0)]
10288          VADCIQ_M))
10289   ]
10290   "TARGET_HAVE_MVE"
10291   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10292   [(set_attr "type" "mve_move")
10293    (set_attr "length" "8")])
10294
10295 ;;
10296 ;; [vadciq_u, vadciq_s])
10297 ;;
10298 (define_insn "mve_vadciq_<supf>v4si"
10299   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10300         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10301                       (match_operand:V4SI 2 "s_register_operand" "w")]
10302          VADCIQ))
10303    (set (reg:SI VFPCC_REGNUM)
10304         (unspec:SI [(const_int 0)]
10305          VADCIQ))
10306   ]
10307   "TARGET_HAVE_MVE"
10308   "vadci.i32\t%q0, %q1, %q2"
10309   [(set_attr "type" "mve_move")
10310    (set_attr "length" "4")])
10311
10312 ;;
10313 ;; [vadcq_m_s, vadcq_m_u])
10314 ;;
10315 (define_insn "mve_vadcq_m_<supf>v4si"
10316   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10317         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10318                       (match_operand:V4SI 2 "s_register_operand" "w")
10319                       (match_operand:V4SI 3 "s_register_operand" "w")
10320                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10321          VADCQ_M))
10322    (set (reg:SI VFPCC_REGNUM)
10323         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10324          VADCQ_M))
10325   ]
10326   "TARGET_HAVE_MVE"
10327   "vpst\;vadct.i32\t%q0, %q2, %q3"
10328   [(set_attr "type" "mve_move")
10329    (set_attr "length" "8")])
10330
10331 ;;
10332 ;; [vadcq_u, vadcq_s])
10333 ;;
10334 (define_insn "mve_vadcq_<supf>v4si"
10335   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10336         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10337                        (match_operand:V4SI 2 "s_register_operand" "w")]
10338          VADCQ))
10339    (set (reg:SI VFPCC_REGNUM)
10340         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10341          VADCQ))
10342   ]
10343   "TARGET_HAVE_MVE"
10344   "vadc.i32\t%q0, %q1, %q2"
10345   [(set_attr "type" "mve_move")
10346    (set_attr "length" "4")
10347    (set_attr "conds" "set")])
10348
10349 ;;
10350 ;; [vsbciq_m_u, vsbciq_m_s])
10351 ;;
10352 (define_insn "mve_vsbciq_m_<supf>v4si"
10353   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10354         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10355                       (match_operand:V4SI 2 "s_register_operand" "w")
10356                       (match_operand:V4SI 3 "s_register_operand" "w")
10357                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10358          VSBCIQ_M))
10359    (set (reg:SI VFPCC_REGNUM)
10360         (unspec:SI [(const_int 0)]
10361          VSBCIQ_M))
10362   ]
10363   "TARGET_HAVE_MVE"
10364   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10365   [(set_attr "type" "mve_move")
10366    (set_attr "length" "8")])
10367
10368 ;;
10369 ;; [vsbciq_s, vsbciq_u])
10370 ;;
10371 (define_insn "mve_vsbciq_<supf>v4si"
10372   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10373         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10374                       (match_operand:V4SI 2 "s_register_operand" "w")]
10375          VSBCIQ))
10376    (set (reg:SI VFPCC_REGNUM)
10377         (unspec:SI [(const_int 0)]
10378          VSBCIQ))
10379   ]
10380   "TARGET_HAVE_MVE"
10381   "vsbci.i32\t%q0, %q1, %q2"
10382   [(set_attr "type" "mve_move")
10383    (set_attr "length" "4")])
10384
10385 ;;
10386 ;; [vsbcq_m_u, vsbcq_m_s])
10387 ;;
10388 (define_insn "mve_vsbcq_m_<supf>v4si"
10389   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10390         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10391                       (match_operand:V4SI 2 "s_register_operand" "w")
10392                       (match_operand:V4SI 3 "s_register_operand" "w")
10393                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10394          VSBCQ_M))
10395    (set (reg:SI VFPCC_REGNUM)
10396         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10397          VSBCQ_M))
10398   ]
10399   "TARGET_HAVE_MVE"
10400   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10401   [(set_attr "type" "mve_move")
10402    (set_attr "length" "8")])
10403
10404 ;;
10405 ;; [vsbcq_s, vsbcq_u])
10406 ;;
10407 (define_insn "mve_vsbcq_<supf>v4si"
10408   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10409         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10410                       (match_operand:V4SI 2 "s_register_operand" "w")]
10411          VSBCQ))
10412    (set (reg:SI VFPCC_REGNUM)
10413         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10414          VSBCQ))
10415   ]
10416   "TARGET_HAVE_MVE"
10417   "vsbc.i32\t%q0, %q1, %q2"
10418   [(set_attr "type" "mve_move")
10419    (set_attr "length" "4")])
10420
10421 ;;
10422 ;; [vst2q])
10423 ;;
10424 (define_insn "mve_vst2q<mode>"
10425   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10426         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10427                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10428          VST2Q))
10429   ]
10430   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10431    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10432 {
10433    rtx ops[4];
10434    int regno = REGNO (operands[1]);
10435    ops[0] = gen_rtx_REG (TImode, regno);
10436    ops[1] = gen_rtx_REG (TImode, regno + 4);
10437    rtx reg  = operands[0];
10438    while (reg && !REG_P (reg))
10439     reg = XEXP (reg, 0);
10440    gcc_assert (REG_P (reg));
10441    ops[2] = reg;
10442    ops[3] = operands[0];
10443    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10444                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10445    return "";
10446 }
10447   [(set_attr "length" "8")])
10448
10449 ;;
10450 ;; [vld2q])
10451 ;;
10452 (define_insn "mve_vld2q<mode>"
10453   [(set (match_operand:OI 0 "s_register_operand" "=w")
10454         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10455                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10456          VLD2Q))
10457   ]
10458   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10459    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10460 {
10461    rtx ops[4];
10462    int regno = REGNO (operands[0]);
10463    ops[0] = gen_rtx_REG (TImode, regno);
10464    ops[1] = gen_rtx_REG (TImode, regno + 4);
10465    rtx reg  = operands[1];
10466    while (reg && !REG_P (reg))
10467     reg = XEXP (reg, 0);
10468    gcc_assert (REG_P (reg));
10469    ops[2] = reg;
10470    ops[3] = operands[1];
10471    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10472                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10473    return "";
10474 }
10475   [(set_attr "length" "8")])
10476
10477 ;;
10478 ;; [vld4q])
10479 ;;
10480 (define_insn "mve_vld4q<mode>"
10481   [(set (match_operand:XI 0 "s_register_operand" "=w")
10482         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10483                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10484          VLD4Q))
10485   ]
10486   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10487    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10488 {
10489    rtx ops[6];
10490    int regno = REGNO (operands[0]);
10491    ops[0] = gen_rtx_REG (TImode, regno);
10492    ops[1] = gen_rtx_REG (TImode, regno+4);
10493    ops[2] = gen_rtx_REG (TImode, regno+8);
10494    ops[3] = gen_rtx_REG (TImode, regno + 12);
10495    rtx reg  = operands[1];
10496    while (reg && !REG_P (reg))
10497     reg = XEXP (reg, 0);
10498    gcc_assert (REG_P (reg));
10499    ops[4] = reg;
10500    ops[5] = operands[1];
10501    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10502                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10503                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10504                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10505    return "";
10506 }
10507   [(set_attr "length" "16")])
10508 ;;
10509 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10510 ;;
10511 (define_insn "mve_vec_extract<mode><V_elem_l>"
10512  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10513    (vec_select:<V_elem>
10514     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10515     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10516   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10517    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10518 {
10519   if (BYTES_BIG_ENDIAN)
10520     {
10521       int elt = INTVAL (operands[2]);
10522       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10523       operands[2] = GEN_INT (elt);
10524     }
10525   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10526 }
10527  [(set_attr "type" "mve_move")])
10528
10529 (define_insn "mve_vec_extractv2didi"
10530  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10531    (vec_select:DI
10532     (match_operand:V2DI 1 "s_register_operand" "w")
10533     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10534   "TARGET_HAVE_MVE"
10535 {
10536   int elt = INTVAL (operands[2]);
10537   if (BYTES_BIG_ENDIAN)
10538     elt = 1 - elt;
10539
10540   if (elt == 0)
10541    return "vmov\t%Q0, %R0, %e1";
10542   else
10543    return "vmov\t%Q0, %R0, %f1";
10544 }
10545  [(set_attr "type" "mve_move")])
10546
10547 (define_insn "*mve_vec_extract_sext_internal<mode>"
10548  [(set (match_operand:SI 0 "s_register_operand" "=r")
10549    (sign_extend:SI
10550     (vec_select:<V_elem>
10551      (match_operand:MVE_2 1 "s_register_operand" "w")
10552      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10553   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10554    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10555 {
10556   if (BYTES_BIG_ENDIAN)
10557     {
10558       int elt = INTVAL (operands[2]);
10559       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10560       operands[2] = GEN_INT (elt);
10561     }
10562   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10563 }
10564  [(set_attr "type" "mve_move")])
10565
10566 (define_insn "*mve_vec_extract_zext_internal<mode>"
10567  [(set (match_operand:SI 0 "s_register_operand" "=r")
10568    (zero_extend:SI
10569     (vec_select:<V_elem>
10570      (match_operand:MVE_2 1 "s_register_operand" "w")
10571      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10572   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10573    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10574 {
10575   if (BYTES_BIG_ENDIAN)
10576     {
10577       int elt = INTVAL (operands[2]);
10578       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10579       operands[2] = GEN_INT (elt);
10580     }
10581   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10582 }
10583  [(set_attr "type" "mve_move")])
10584
10585 ;;
10586 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10587 ;;
10588 (define_insn "mve_vec_set<mode>_internal"
10589  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10590        (vec_merge:VQ2
10591         (vec_duplicate:VQ2
10592           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10593         (match_operand:VQ2 3 "s_register_operand" "0")
10594         (match_operand:SI 2 "immediate_operand" "i")))]
10595   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10596    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10597 {
10598   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10599   if (BYTES_BIG_ENDIAN)
10600     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10601   operands[2] = GEN_INT (elt);
10602
10603   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10604 }
10605  [(set_attr "type" "mve_move")])
10606
10607 (define_insn "mve_vec_setv2di_internal"
10608  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10609        (vec_merge:V2DI
10610         (vec_duplicate:V2DI
10611           (match_operand:DI 1 "nonimmediate_operand" "r"))
10612         (match_operand:V2DI 3 "s_register_operand" "0")
10613         (match_operand:SI 2 "immediate_operand" "i")))]
10614  "TARGET_HAVE_MVE"
10615 {
10616   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10617   if (BYTES_BIG_ENDIAN)
10618     elt = 1 - elt;
10619
10620   if (elt == 0)
10621    return "vmov\t%e0, %Q1, %R1";
10622   else
10623    return "vmov\t%f0, %J1, %K1";
10624 }
10625  [(set_attr "type" "mve_move")])
10626
10627 ;;
10628 ;; [uqrshll_di]
10629 ;;
10630 (define_insn "mve_uqrshll_sat<supf>_di"
10631   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10632         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10633                     (match_operand:SI 2 "register_operand" "r")]
10634          UQRSHLLQ))]
10635   "TARGET_HAVE_MVE"
10636   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10637   [(set_attr "predicable" "yes")])
10638
10639 ;;
10640 ;; [sqrshrl_di]
10641 ;;
10642 (define_insn "mve_sqrshrl_sat<supf>_di"
10643   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10644         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10645                     (match_operand:SI 2 "register_operand" "r")]
10646          SQRSHRLQ))]
10647   "TARGET_HAVE_MVE"
10648   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10649   [(set_attr "predicable" "yes")])
10650
10651 ;;
10652 ;; [uqrshl_si]
10653 ;;
10654 (define_insn "mve_uqrshl_si"
10655   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10656         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10657                     (match_operand:SI 2 "register_operand" "r")]
10658          UQRSHL))]
10659   "TARGET_HAVE_MVE"
10660   "uqrshl%?\\t%1, %2"
10661   [(set_attr "predicable" "yes")])
10662
10663 ;;
10664 ;; [sqrshr_si]
10665 ;;
10666 (define_insn "mve_sqrshr_si"
10667   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10668         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10669                     (match_operand:SI 2 "register_operand" "r")]
10670          SQRSHR))]
10671   "TARGET_HAVE_MVE"
10672   "sqrshr%?\\t%1, %2"
10673   [(set_attr "predicable" "yes")])
10674
10675 ;;
10676 ;; [uqshll_di]
10677 ;;
10678 (define_insn "mve_uqshll_di"
10679   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10680         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10681                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10682   "TARGET_HAVE_MVE"
10683   "uqshll%?\\t%Q1, %R1, %2"
10684   [(set_attr "predicable" "yes")])
10685
10686 ;;
10687 ;; [urshrl_di]
10688 ;;
10689 (define_insn "mve_urshrl_di"
10690   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10691         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10692                     (match_operand:SI 2 "immediate_operand" "Pg")]
10693          URSHRL))]
10694   "TARGET_HAVE_MVE"
10695   "urshrl%?\\t%Q1, %R1, %2"
10696   [(set_attr "predicable" "yes")])
10697
10698 ;;
10699 ;; [uqshl_si]
10700 ;;
10701 (define_insn "mve_uqshl_si"
10702   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10703         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10704                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10705   "TARGET_HAVE_MVE"
10706   "uqshl%?\\t%1, %2"
10707   [(set_attr "predicable" "yes")])
10708
10709 ;;
10710 ;; [urshr_si]
10711 ;;
10712 (define_insn "mve_urshr_si"
10713   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10714         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10715                     (match_operand:SI 2 "immediate_operand" "Pg")]
10716          URSHR))]
10717   "TARGET_HAVE_MVE"
10718   "urshr%?\\t%1, %2"
10719   [(set_attr "predicable" "yes")])
10720
10721 ;;
10722 ;; [sqshl_si]
10723 ;;
10724 (define_insn "mve_sqshl_si"
10725   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10726         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10727                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10728   "TARGET_HAVE_MVE"
10729   "sqshl%?\\t%1, %2"
10730   [(set_attr "predicable" "yes")])
10731
10732 ;;
10733 ;; [srshr_si]
10734 ;;
10735 (define_insn "mve_srshr_si"
10736   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10737         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10738                     (match_operand:SI 2 "immediate_operand" "Pg")]
10739          SRSHR))]
10740   "TARGET_HAVE_MVE"
10741   "srshr%?\\t%1, %2"
10742   [(set_attr "predicable" "yes")])
10743
10744 ;;
10745 ;; [srshrl_di]
10746 ;;
10747 (define_insn "mve_srshrl_di"
10748   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10749         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10750                     (match_operand:SI 2 "immediate_operand" "Pg")]
10751          SRSHRL))]
10752   "TARGET_HAVE_MVE"
10753   "srshrl%?\\t%Q1, %R1, %2"
10754   [(set_attr "predicable" "yes")])
10755
10756 ;;
10757 ;; [sqshll_di]
10758 ;;
10759 (define_insn "mve_sqshll_di"
10760   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10761         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10762                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10763   "TARGET_HAVE_MVE"
10764   "sqshll%?\\t%Q1, %R1, %2"
10765   [(set_attr "predicable" "yes")])
10766
10767 ;;
10768 ;; [vshlcq_m_u vshlcq_m_s]
10769 ;;
10770 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10771  [(match_operand:MVE_2 0 "s_register_operand")
10772   (match_operand:MVE_2 1 "s_register_operand")
10773   (match_operand:SI 2 "s_register_operand")
10774   (match_operand:SI 3 "mve_imm_32")
10775   (match_operand:HI 4 "vpr_register_operand")
10776   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10777  "TARGET_HAVE_MVE"
10778 {
10779   rtx ignore_wb = gen_reg_rtx (SImode);
10780   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10781                                             operands[2], operands[3],
10782                                             operands[4]));
10783   DONE;
10784 })
10785
10786 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10787  [(match_operand:SI 0 "s_register_operand")
10788   (match_operand:MVE_2 1 "s_register_operand")
10789   (match_operand:SI 2 "s_register_operand")
10790   (match_operand:SI 3 "mve_imm_32")
10791   (match_operand:HI 4 "vpr_register_operand")
10792   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10793  "TARGET_HAVE_MVE"
10794 {
10795   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10796   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10797                                             operands[1], operands[2],
10798                                             operands[3], operands[4]));
10799   DONE;
10800 })
10801
10802 (define_insn "mve_vshlcq_m_<supf><mode>"
10803  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10804        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10805                       (match_operand:SI 3 "s_register_operand" "1")
10806                       (match_operand:SI 4 "mve_imm_32" "Rf")
10807                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10808         VSHLCQ_M))
10809   (set (match_operand:SI  1 "s_register_operand" "=r")
10810        (unspec:SI [(match_dup 2)
10811                    (match_dup 3)
10812                    (match_dup 4)
10813                    (match_dup 5)]
10814         VSHLCQ_M))
10815  ]
10816  "TARGET_HAVE_MVE"
10817  "vpst\;vshlct\t%q0, %1, %4"
10818  [(set_attr "type" "mve_move")
10819   (set_attr "length" "8")])
10820
10821 (define_insn "*mve_vec_duplicate<mode>"
10822  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
10823        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
10824  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
10825  "vdup.<V_sz_elem>\t%q0, %1"
10826  [(set_attr "type" "mve_move")])
10827
10828 ;; CDE instructions on MVE registers.
10829
10830 (define_insn "arm_vcx1qv16qi"
10831   [(set (match_operand:V16QI 0 "register_operand" "=t")
10832         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10833                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10834          UNSPEC_VCDE))]
10835   "TARGET_CDE && TARGET_HAVE_MVE"
10836   "vcx1\\tp%c1, %q0, #%c2"
10837   [(set_attr "type" "coproc")]
10838 )
10839
10840 (define_insn "arm_vcx1qav16qi"
10841   [(set (match_operand:V16QI 0 "register_operand" "=t")
10842         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10843                             (match_operand:V16QI 2 "register_operand" "0")
10844                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10845          UNSPEC_VCDEA))]
10846   "TARGET_CDE && TARGET_HAVE_MVE"
10847   "vcx1a\\tp%c1, %q0, #%c3"
10848   [(set_attr "type" "coproc")]
10849 )
10850
10851 (define_insn "arm_vcx2qv16qi"
10852   [(set (match_operand:V16QI 0 "register_operand" "=t")
10853         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10854                           (match_operand:V16QI 2 "register_operand" "t")
10855                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10856          UNSPEC_VCDE))]
10857   "TARGET_CDE && TARGET_HAVE_MVE"
10858   "vcx2\\tp%c1, %q0, %q2, #%c3"
10859   [(set_attr "type" "coproc")]
10860 )
10861
10862 (define_insn "arm_vcx2qav16qi"
10863   [(set (match_operand:V16QI 0 "register_operand" "=t")
10864         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10865                           (match_operand:V16QI 2 "register_operand" "0")
10866                           (match_operand:V16QI 3 "register_operand" "t")
10867                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10868          UNSPEC_VCDEA))]
10869   "TARGET_CDE && TARGET_HAVE_MVE"
10870   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10871   [(set_attr "type" "coproc")]
10872 )
10873
10874 (define_insn "arm_vcx3qv16qi"
10875   [(set (match_operand:V16QI 0 "register_operand" "=t")
10876         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10877                           (match_operand:V16QI 2 "register_operand" "t")
10878                           (match_operand:V16QI 3 "register_operand" "t")
10879                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10880          UNSPEC_VCDE))]
10881   "TARGET_CDE && TARGET_HAVE_MVE"
10882   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10883   [(set_attr "type" "coproc")]
10884 )
10885
10886 (define_insn "arm_vcx3qav16qi"
10887   [(set (match_operand:V16QI 0 "register_operand" "=t")
10888         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10889                           (match_operand:V16QI 2 "register_operand" "0")
10890                           (match_operand:V16QI 3 "register_operand" "t")
10891                           (match_operand:V16QI 4 "register_operand" "t")
10892                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10893          UNSPEC_VCDEA))]
10894   "TARGET_CDE && TARGET_HAVE_MVE"
10895   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10896   [(set_attr "type" "coproc")]
10897 )
10898
10899 (define_insn "arm_vcx1q<a>_p_v16qi"
10900   [(set (match_operand:V16QI 0 "register_operand" "=t")
10901         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10902                            (match_operand:V16QI 2 "register_operand" "0")
10903                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10904                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10905          CDE_VCX))]
10906   "TARGET_CDE && TARGET_HAVE_MVE"
10907   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10908   [(set_attr "type" "coproc")
10909    (set_attr "length" "8")]
10910 )
10911
10912 (define_insn "arm_vcx2q<a>_p_v16qi"
10913   [(set (match_operand:V16QI 0 "register_operand" "=t")
10914         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10915                           (match_operand:V16QI 2 "register_operand" "0")
10916                           (match_operand:V16QI 3 "register_operand" "t")
10917                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10918                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10919          CDE_VCX))]
10920   "TARGET_CDE && TARGET_HAVE_MVE"
10921   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10922   [(set_attr "type" "coproc")
10923    (set_attr "length" "8")]
10924 )
10925
10926 (define_insn "arm_vcx3q<a>_p_v16qi"
10927   [(set (match_operand:V16QI 0 "register_operand" "=t")
10928         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10929                           (match_operand:V16QI 2 "register_operand" "0")
10930                           (match_operand:V16QI 3 "register_operand" "t")
10931                           (match_operand:V16QI 4 "register_operand" "t")
10932                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10933                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10934          CDE_VCX))]
10935   "TARGET_CDE && TARGET_HAVE_MVE"
10936   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10937   [(set_attr "type" "coproc")
10938    (set_attr "length" "8")]
10939 )