gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2021 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Uxi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       rtx ops[2];
  45       int regno = (which_alternative == 7)
  46                   ? REGNO (operands[1]) : REGNO (operands[0]);
  47
  48       ops[0] = operands[0];
  49       ops[1] = operands[1];
  50       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode)
  51         {
  52           if (which_alternative == 7)
  53             {
  54               ops[1] = gen_rtx_REG (DImode, regno);
  55               output_asm_insn ("vstr.64\t%P1, %E0",ops);
  56             }
  57           else
  58             {
  59               ops[0] = gen_rtx_REG (DImode, regno);
  60               output_asm_insn ("vldr.64\t%P0, %E1",ops);
  61             }
  62         }
  63       else if (<MODE>mode == TImode)
  64         {
  65           if (which_alternative == 7)
  66             output_asm_insn ("vstr.64\t%q1, %E0",ops);
  67           else
  68             output_asm_insn ("vldr.64\t%q0, %E1",ops);
  69         }
  70       else
  71         {
  72           if (which_alternative == 7)
  73             {
  74               ops[1] = gen_rtx_REG (TImode, regno);
  75               output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0",ops);
  76             }
  77           else
  78             {
  79               ops[0] = gen_rtx_REG (TImode, regno);
  80               output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1",ops);
  81             }
  82         }
  83       return "";
  84     }
  85   switch (which_alternative)
  86     {
  87     case 0:
  88       return "vmov\t%q0, %q1";
  89     case 1:
  90       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  91     case 2:
  92       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  93     case 5:
  94       return output_move_quad (operands);
  95     case 8:
  96         return output_move_neon (operands);
  97     default:
  98       gcc_unreachable ();
  99       return "";
 100     }
 101 }
 102   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
 103    (set_attr "length" "4,8,8,4,8,8,4,4,4")
 104    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
 105    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
 106
 107 (define_insn "*mve_mov<mode>"
 108   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 109         (vec_duplicate:MVE_types
 110           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 111   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 112 {
 113   if (which_alternative == 0)
 114     return "vdup.<V_sz_elem>\t%q0, %1";
 115   return "vmov.<V_sz_elem>\t%q0, %1";
 116 }
 117   [(set_attr "length" "4,4")
 118    (set_attr "type" "mve_move,mve_move")])
 119
 120 ;;
 121 ;; [vst4q])
 122 ;;
 123 (define_insn "mve_vst4q<mode>"
 124   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 125         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 126                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 127          VST4Q))
 128   ]
 129   "TARGET_HAVE_MVE"
 130 {
 131    rtx ops[6];
 132    int regno = REGNO (operands[1]);
 133    ops[0] = gen_rtx_REG (TImode, regno);
 134    ops[1] = gen_rtx_REG (TImode, regno+4);
 135    ops[2] = gen_rtx_REG (TImode, regno+8);
 136    ops[3] = gen_rtx_REG (TImode, regno+12);
 137    rtx reg  = operands[0];
 138    while (reg && !REG_P (reg))
 139     reg = XEXP (reg, 0);
 140    gcc_assert (REG_P (reg));
 141    ops[4] = reg;
 142    ops[5] = operands[0];
 143    /* Here in first three instructions data is stored to ops[4]'s location but
 144       in the fourth instruction data is stored to operands[0], this is to
 145       support the writeback.  */
 146    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 147                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 148                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 149                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 150    return "";
 151 }
 152   [(set_attr "length" "16")])
 153
 154 ;;
 155 ;; [vrndq_m_f])
 156 ;;
 157 (define_insn "mve_vrndq_m_f<mode>"
 158   [
 159    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 160         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 161                        (match_operand:MVE_0 2 "s_register_operand" "w")
 162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 163          VRNDQ_M_F))
 164   ]
 165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 166   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 167   [(set_attr "type" "mve_move")
 168    (set_attr "length""8")])
 169
 170 ;;
 171 ;; [vrndxq_f])
 172 ;;
 173 (define_insn "mve_vrndxq_f<mode>"
 174   [
 175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 177          VRNDXQ_F))
 178   ]
 179   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 180   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 181   [(set_attr "type" "mve_move")
 182 ])
 183
 184 ;;
 185 ;; [vrndq_f])
 186 ;;
 187 (define_insn "mve_vrndq_f<mode>"
 188   [
 189    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 190         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 191          VRNDQ_F))
 192   ]
 193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 194   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 195   [(set_attr "type" "mve_move")
 196 ])
 197
 198 ;;
 199 ;; [vrndpq_f])
 200 ;;
 201 (define_insn "mve_vrndpq_f<mode>"
 202   [
 203    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 204         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 205          VRNDPQ_F))
 206   ]
 207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 208   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 209   [(set_attr "type" "mve_move")
 210 ])
 211
 212 ;;
 213 ;; [vrndnq_f])
 214 ;;
 215 (define_insn "mve_vrndnq_f<mode>"
 216   [
 217    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 218         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 219          VRNDNQ_F))
 220   ]
 221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 222   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 223   [(set_attr "type" "mve_move")
 224 ])
 225
 226 ;;
 227 ;; [vrndmq_f])
 228 ;;
 229 (define_insn "mve_vrndmq_f<mode>"
 230   [
 231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 232         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 233          VRNDMQ_F))
 234   ]
 235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 236   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 237   [(set_attr "type" "mve_move")
 238 ])
 239
 240 ;;
 241 ;; [vrndaq_f])
 242 ;;
 243 (define_insn "mve_vrndaq_f<mode>"
 244   [
 245    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 246         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 247          VRNDAQ_F))
 248   ]
 249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 250   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 251   [(set_attr "type" "mve_move")
 252 ])
 253
 254 ;;
 255 ;; [vrev64q_f])
 256 ;;
 257 (define_insn "mve_vrev64q_f<mode>"
 258   [
 259    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 260         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 261          VREV64Q_F))
 262   ]
 263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 264   "vrev64.%#<V_sz_elem> %q0, %q1"
 265   [(set_attr "type" "mve_move")
 266 ])
 267
 268 ;;
 269 ;; [vnegq_f])
 270 ;;
 271 (define_insn "mve_vnegq_f<mode>"
 272   [
 273    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 274         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 275   ]
 276   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 277   "vneg.f%#<V_sz_elem>  %q0, %q1"
 278   [(set_attr "type" "mve_move")
 279 ])
 280
 281 ;;
 282 ;; [vdupq_n_f])
 283 ;;
 284 (define_insn "mve_vdupq_n_f<mode>"
 285   [
 286    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 287         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 288          VDUPQ_N_F))
 289   ]
 290   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 291   "vdup.%#<V_sz_elem>   %q0, %1"
 292   [(set_attr "type" "mve_move")
 293 ])
 294
 295 ;;
 296 ;; [vabsq_f])
 297 ;;
 298 (define_insn "mve_vabsq_f<mode>"
 299   [
 300    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 301         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 302          VABSQ_F))
 303   ]
 304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 305   "vabs.f%#<V_sz_elem>  %q0, %q1"
 306   [(set_attr "type" "mve_move")
 307 ])
 308
 309 ;;
 310 ;; [vrev32q_f])
 311 ;;
 312 (define_insn "mve_vrev32q_fv8hf"
 313   [
 314    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 315         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 316          VREV32Q_F))
 317   ]
 318   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 319   "vrev32.16 %q0, %q1"
 320   [(set_attr "type" "mve_move")
 321 ])
 322 ;;
 323 ;; [vcvttq_f32_f16])
 324 ;;
 325 (define_insn "mve_vcvttq_f32_f16v4sf"
 326   [
 327    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 328         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 329          VCVTTQ_F32_F16))
 330   ]
 331   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 332   "vcvtt.f32.f16 %q0, %q1"
 333   [(set_attr "type" "mve_move")
 334 ])
 335
 336 ;;
 337 ;; [vcvtbq_f32_f16])
 338 ;;
 339 (define_insn "mve_vcvtbq_f32_f16v4sf"
 340   [
 341    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 342         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 343          VCVTBQ_F32_F16))
 344   ]
 345   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 346   "vcvtb.f32.f16 %q0, %q1"
 347   [(set_attr "type" "mve_move")
 348 ])
 349
 350 ;;
 351 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 352 ;;
 353 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 354   [
 355    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 356         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 357          VCVTQ_TO_F))
 358   ]
 359   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 360   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 361   [(set_attr "type" "mve_move")
 362 ])
 363
 364 ;;
 365 ;; [vrev64q_u, vrev64q_s])
 366 ;;
 367 (define_insn "mve_vrev64q_<supf><mode>"
 368   [
 369    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 370         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 371          VREV64Q))
 372   ]
 373   "TARGET_HAVE_MVE"
 374   "vrev64.%#<V_sz_elem> %q0, %q1"
 375   [(set_attr "type" "mve_move")
 376 ])
 377
 378 ;;
 379 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 380 ;;
 381 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 382   [
 383    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 384         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 385          VCVTQ_FROM_F))
 386   ]
 387   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 388   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 389   [(set_attr "type" "mve_move")
 390 ])
 391 ;; [vqnegq_s])
 392 ;;
 393 (define_insn "mve_vqnegq_s<mode>"
 394   [
 395    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 396         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 397          VQNEGQ_S))
 398   ]
 399   "TARGET_HAVE_MVE"
 400   "vqneg.s%#<V_sz_elem> %q0, %q1"
 401   [(set_attr "type" "mve_move")
 402 ])
 403
 404 ;;
 405 ;; [vqabsq_s])
 406 ;;
 407 (define_insn "mve_vqabsq_s<mode>"
 408   [
 409    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 410         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 411          VQABSQ_S))
 412   ]
 413   "TARGET_HAVE_MVE"
 414   "vqabs.s%#<V_sz_elem> %q0, %q1"
 415   [(set_attr "type" "mve_move")
 416 ])
 417
 418 ;;
 419 ;; [vnegq_s])
 420 ;;
 421 (define_insn "mve_vnegq_s<mode>"
 422   [
 423    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 424         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 425   ]
 426   "TARGET_HAVE_MVE"
 427   "vneg.s%#<V_sz_elem>  %q0, %q1"
 428   [(set_attr "type" "mve_move")
 429 ])
 430
 431 ;;
 432 ;; [vmvnq_u, vmvnq_s])
 433 ;;
 434 (define_insn "mve_vmvnq_u<mode>"
 435   [
 436    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 437         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 438   ]
 439   "TARGET_HAVE_MVE"
 440   "vmvn\t%q0, %q1"
 441   [(set_attr "type" "mve_move")
 442 ])
 443 (define_expand "mve_vmvnq_s<mode>"
 444   [
 445    (set (match_operand:MVE_2 0 "s_register_operand")
 446         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 447   ]
 448   "TARGET_HAVE_MVE"
 449 )
 450
 451 ;;
 452 ;; [vdupq_n_u, vdupq_n_s])
 453 ;;
 454 (define_insn "mve_vdupq_n_<supf><mode>"
 455   [
 456    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 457         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 458          VDUPQ_N))
 459   ]
 460   "TARGET_HAVE_MVE"
 461   "vdup.%#<V_sz_elem>   %q0, %1"
 462   [(set_attr "type" "mve_move")
 463 ])
 464
 465 ;;
 466 ;; [vclzq_u, vclzq_s])
 467 ;;
 468 (define_insn "mve_vclzq_<supf><mode>"
 469   [
 470    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 471         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 472          VCLZQ))
 473   ]
 474   "TARGET_HAVE_MVE"
 475   "vclz.i%#<V_sz_elem>  %q0, %q1"
 476   [(set_attr "type" "mve_move")
 477 ])
 478
 479 ;;
 480 ;; [vclsq_s])
 481 ;;
 482 (define_insn "mve_vclsq_s<mode>"
 483   [
 484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 486          VCLSQ_S))
 487   ]
 488   "TARGET_HAVE_MVE"
 489   "vcls.s%#<V_sz_elem>  %q0, %q1"
 490   [(set_attr "type" "mve_move")
 491 ])
 492
 493 ;;
 494 ;; [vaddvq_s, vaddvq_u])
 495 ;;
 496 (define_insn "mve_vaddvq_<supf><mode>"
 497   [
 498    (set (match_operand:SI 0 "s_register_operand" "=Te")
 499         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 500          VADDVQ))
 501   ]
 502   "TARGET_HAVE_MVE"
 503   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 504   [(set_attr "type" "mve_move")
 505 ])
 506
 507 ;;
 508 ;; [vabsq_s])
 509 ;;
 510 (define_insn "mve_vabsq_s<mode>"
 511   [
 512    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 513         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 514          VABSQ_S))
 515   ]
 516   "TARGET_HAVE_MVE"
 517   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 518   [(set_attr "type" "mve_move")
 519 ])
 520
 521 ;;
 522 ;; [vrev32q_u, vrev32q_s])
 523 ;;
 524 (define_insn "mve_vrev32q_<supf><mode>"
 525   [
 526    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 527         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 528          VREV32Q))
 529   ]
 530   "TARGET_HAVE_MVE"
 531   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 532   [(set_attr "type" "mve_move")
 533 ])
 534
 535 ;;
 536 ;; [vmovltq_u, vmovltq_s])
 537 ;;
 538 (define_insn "mve_vmovltq_<supf><mode>"
 539   [
 540    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 541         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 542          VMOVLTQ))
 543   ]
 544   "TARGET_HAVE_MVE"
 545   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 546   [(set_attr "type" "mve_move")
 547 ])
 548
 549 ;;
 550 ;; [vmovlbq_s, vmovlbq_u])
 551 ;;
 552 (define_insn "mve_vmovlbq_<supf><mode>"
 553   [
 554    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 555         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 556          VMOVLBQ))
 557   ]
 558   "TARGET_HAVE_MVE"
 559   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 560   [(set_attr "type" "mve_move")
 561 ])
 562
 563 ;;
 564 ;; [vcvtpq_s, vcvtpq_u])
 565 ;;
 566 (define_insn "mve_vcvtpq_<supf><mode>"
 567   [
 568    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 569         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 570          VCVTPQ))
 571   ]
 572   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 573   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 574   [(set_attr "type" "mve_move")
 575 ])
 576
 577 ;;
 578 ;; [vcvtnq_s, vcvtnq_u])
 579 ;;
 580 (define_insn "mve_vcvtnq_<supf><mode>"
 581   [
 582    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 583         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 584          VCVTNQ))
 585   ]
 586   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 587   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 588   [(set_attr "type" "mve_move")
 589 ])
 590
 591 ;;
 592 ;; [vcvtmq_s, vcvtmq_u])
 593 ;;
 594 (define_insn "mve_vcvtmq_<supf><mode>"
 595   [
 596    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 597         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 598          VCVTMQ))
 599   ]
 600   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 601   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 602   [(set_attr "type" "mve_move")
 603 ])
 604
 605 ;;
 606 ;; [vcvtaq_u, vcvtaq_s])
 607 ;;
 608 (define_insn "mve_vcvtaq_<supf><mode>"
 609   [
 610    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 611         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 612          VCVTAQ))
 613   ]
 614   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 615   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 616   [(set_attr "type" "mve_move")
 617 ])
 618
 619 ;;
 620 ;; [vmvnq_n_u, vmvnq_n_s])
 621 ;;
 622 (define_insn "mve_vmvnq_n_<supf><mode>"
 623   [
 624    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 625         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 626          VMVNQ_N))
 627   ]
 628   "TARGET_HAVE_MVE"
 629   "vmvn.i%#<V_sz_elem>  %q0, %1"
 630   [(set_attr "type" "mve_move")
 631 ])
 632
 633 ;;
 634 ;; [vrev16q_u, vrev16q_s])
 635 ;;
 636 (define_insn "mve_vrev16q_<supf>v16qi"
 637   [
 638    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 639         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 640          VREV16Q))
 641   ]
 642   "TARGET_HAVE_MVE"
 643   "vrev16.8 %q0, %q1"
 644   [(set_attr "type" "mve_move")
 645 ])
 646
 647 ;;
 648 ;; [vaddlvq_s vaddlvq_u])
 649 ;;
 650 (define_insn "mve_vaddlvq_<supf>v4si"
 651   [
 652    (set (match_operand:DI 0 "s_register_operand" "=r")
 653         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 654          VADDLVQ))
 655   ]
 656   "TARGET_HAVE_MVE"
 657   "vaddlv.<supf>32 %Q0, %R0, %q1"
 658   [(set_attr "type" "mve_move")
 659 ])
 660
 661 ;;
 662 ;; [vctp8q vctp16q vctp32q vctp64q])
 663 ;;
 664 (define_insn "mve_vctp<mode1>qhi"
 665   [
 666    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 667         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 668         VCTPQ))
 669   ]
 670   "TARGET_HAVE_MVE"
 671   "vctp.<mode1> %1"
 672   [(set_attr "type" "mve_move")
 673 ])
 674
 675 ;;
 676 ;; [vpnot])
 677 ;;
 678 (define_insn "mve_vpnothi"
 679   [
 680    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 681         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 682          VPNOT))
 683   ]
 684   "TARGET_HAVE_MVE"
 685   "vpnot"
 686   [(set_attr "type" "mve_move")
 687 ])
 688
 689 ;;
 690 ;; [vsubq_n_f])
 691 ;;
 692 (define_insn "mve_vsubq_n_f<mode>"
 693   [
 694    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 695         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 696                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 697          VSUBQ_N_F))
 698   ]
 699   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 700   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 701   [(set_attr "type" "mve_move")
 702 ])
 703
 704 ;;
 705 ;; [vbrsrq_n_f])
 706 ;;
 707 (define_insn "mve_vbrsrq_n_f<mode>"
 708   [
 709    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 710         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 711                        (match_operand:SI 2 "s_register_operand" "r")]
 712          VBRSRQ_N_F))
 713   ]
 714   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 715   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 716   [(set_attr "type" "mve_move")
 717 ])
 718
 719 ;;
 720 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 721 ;;
 722 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 723   [
 724    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 725         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 726                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 727          VCVTQ_N_TO_F))
 728   ]
 729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 730   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 731   [(set_attr "type" "mve_move")
 732 ])
 733
 734 ;; [vcreateq_f])
 735 ;;
 736 (define_insn "mve_vcreateq_f<mode>"
 737   [
 738    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 739         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 740                        (match_operand:DI 2 "s_register_operand" "r")]
 741          VCREATEQ_F))
 742   ]
 743   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 744   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 745   [(set_attr "type" "mve_move")
 746    (set_attr "length""8")])
 747
 748 ;;
 749 ;; [vcreateq_u, vcreateq_s])
 750 ;;
 751 (define_insn "mve_vcreateq_<supf><mode>"
 752   [
 753    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 754         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 755                        (match_operand:DI 2 "s_register_operand" "r")]
 756          VCREATEQ))
 757   ]
 758   "TARGET_HAVE_MVE"
 759   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 760   [(set_attr "type" "mve_move")
 761    (set_attr "length""8")])
 762
 763 ;;
 764 ;; [vshrq_n_s, vshrq_n_u])
 765 ;;
 766 ;; Version that takes an immediate as operand 2.
 767 (define_insn "mve_vshrq_n_<supf><mode>"
 768   [
 769    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 770         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 771                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 772          VSHRQ_N))
 773   ]
 774   "TARGET_HAVE_MVE"
 775   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 776   [(set_attr "type" "mve_move")
 777 ])
 778
 779 ;; Versions that take constant vectors as operand 2 (with all elements
 780 ;; equal).
 781 (define_insn "mve_vshrq_n_s<mode>_imm"
 782   [
 783    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 784         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 785                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 786   ]
 787   "TARGET_HAVE_MVE"
 788   {
 789     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 790                                         <MODE>mode,
 791                                         VALID_NEON_QREG_MODE (<MODE>mode),
 792                                         true);
 793   }
 794   [(set_attr "type" "mve_move")
 795 ])
 796 (define_insn "mve_vshrq_n_u<mode>_imm"
 797   [
 798    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 799         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 800                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 801   ]
 802   "TARGET_HAVE_MVE"
 803   {
 804     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 805                                         <MODE>mode,
 806                                         VALID_NEON_QREG_MODE (<MODE>mode),
 807                                         true);
 808   }
 809   [(set_attr "type" "mve_move")
 810 ])
 811
 812 ;;
 813 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 814 ;;
 815 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 816   [
 817    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 818         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 819                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 820          VCVTQ_N_FROM_F))
 821   ]
 822   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 823   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 824   [(set_attr "type" "mve_move")
 825 ])
 826
 827 ;;
 828 ;; [vaddlvq_p_s])
 829 ;;
 830 (define_insn "mve_vaddlvq_p_<supf>v4si"
 831   [
 832    (set (match_operand:DI 0 "s_register_operand" "=r")
 833         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 834                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 835          VADDLVQ_P))
 836   ]
 837   "TARGET_HAVE_MVE"
 838   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 839   [(set_attr "type" "mve_move")
 840    (set_attr "length""8")])
 841
 842 ;;
 843 ;; [vcmpneq_u, vcmpneq_s])
 844 ;;
 845 (define_insn "mve_vcmpneq_<supf><mode>"
 846   [
 847    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 848         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 849                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 850          VCMPNEQ))
 851   ]
 852   "TARGET_HAVE_MVE"
 853   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
 854   [(set_attr "type" "mve_move")
 855 ])
 856
 857 ;;
 858 ;; [vshlq_s, vshlq_u])
 859 ;; See vec-common.md
 860
 861 ;;
 862 ;; [vabdq_s, vabdq_u])
 863 ;;
 864 (define_insn "mve_vabdq_<supf><mode>"
 865   [
 866    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 867         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 868                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 869          VABDQ))
 870   ]
 871   "TARGET_HAVE_MVE"
 872   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 873   [(set_attr "type" "mve_move")
 874 ])
 875
 876 ;;
 877 ;; [vaddq_n_s, vaddq_n_u])
 878 ;;
 879 (define_insn "mve_vaddq_n_<supf><mode>"
 880   [
 881    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 882         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 883                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 884          VADDQ_N))
 885   ]
 886   "TARGET_HAVE_MVE"
 887   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 888   [(set_attr "type" "mve_move")
 889 ])
 890
 891 ;;
 892 ;; [vaddvaq_s, vaddvaq_u])
 893 ;;
 894 (define_insn "mve_vaddvaq_<supf><mode>"
 895   [
 896    (set (match_operand:SI 0 "s_register_operand" "=Te")
 897         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 898                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 899          VADDVAQ))
 900   ]
 901   "TARGET_HAVE_MVE"
 902   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 903   [(set_attr "type" "mve_move")
 904 ])
 905
 906 ;;
 907 ;; [vaddvq_p_u, vaddvq_p_s])
 908 ;;
 909 (define_insn "mve_vaddvq_p_<supf><mode>"
 910   [
 911    (set (match_operand:SI 0 "s_register_operand" "=Te")
 912         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 913                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 914          VADDVQ_P))
 915   ]
 916   "TARGET_HAVE_MVE"
 917   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 918   [(set_attr "type" "mve_move")
 919    (set_attr "length""8")])
 920
 921 ;;
 922 ;; [vandq_u, vandq_s])
 923 ;;
 924 ;; signed and unsigned versions are the same: define the unsigned
 925 ;; insn, and use an expander for the signed one as we still reference
 926 ;; both names from arm_mve.h.
 927 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 928 (define_insn "mve_vandq_u<mode>"
 929   [
 930    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 931         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 932                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 933   ]
 934   "TARGET_HAVE_MVE"
 935   "@
 936    vand\t%q0, %q1, %q2
 937    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 938   [(set_attr "type" "mve_move")
 939 ])
 940 (define_expand "mve_vandq_s<mode>"
 941   [
 942    (set (match_operand:MVE_2 0 "s_register_operand")
 943         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 944                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 945   ]
 946   "TARGET_HAVE_MVE"
 947 )
 948
 949 ;;
 950 ;; [vbicq_s, vbicq_u])
 951 ;;
 952 (define_insn "mve_vbicq_u<mode>"
 953   [
 954    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 955         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 956                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 957   ]
 958   "TARGET_HAVE_MVE"
 959   "vbic\t%q0, %q1, %q2"
 960   [(set_attr "type" "mve_move")
 961 ])
 962
 963 (define_expand "mve_vbicq_s<mode>"
 964   [
 965    (set (match_operand:MVE_2 0 "s_register_operand")
 966         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 967                    (match_operand:MVE_2 1 "s_register_operand")))
 968   ]
 969   "TARGET_HAVE_MVE"
 970 )
 971
 972 ;;
 973 ;; [vbrsrq_n_u, vbrsrq_n_s])
 974 ;;
 975 (define_insn "mve_vbrsrq_n_<supf><mode>"
 976   [
 977    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 978         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 979                        (match_operand:SI 2 "s_register_operand" "r")]
 980          VBRSRQ_N))
 981   ]
 982   "TARGET_HAVE_MVE"
 983   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 984   [(set_attr "type" "mve_move")
 985 ])
 986
 987 ;;
 988 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 989 ;;
 990 (define_insn "mve_vcaddq<mve_rot><mode>"
 991   [
 992    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 993         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 994                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 995          VCADD))
 996   ]
 997   "TARGET_HAVE_MVE"
 998   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 999   [(set_attr "type" "mve_move")
1000 ])
1001
1002 ;; Auto vectorizer pattern for int vcadd
1003 (define_expand "cadd<rot><mode>3"
1004   [(set (match_operand:MVE_2 0 "register_operand")
1005         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
1006                        (match_operand:MVE_2 2 "register_operand")]
1007           VCADD))]
1008   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
1009 )
1010
1011 ;;
1012 ;; [vcmpcsq_n_u])
1013 ;;
1014 (define_insn "mve_vcmpcsq_n_u<mode>"
1015   [
1016    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1017         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1018                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1019          VCMPCSQ_N_U))
1020   ]
1021   "TARGET_HAVE_MVE"
1022   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1023   [(set_attr "type" "mve_move")
1024 ])
1025
1026 ;;
1027 ;; [vcmpcsq_u])
1028 ;;
1029 (define_insn "mve_vcmpcsq_u<mode>"
1030   [
1031    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1032         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1033                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1034          VCMPCSQ_U))
1035   ]
1036   "TARGET_HAVE_MVE"
1037   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1038   [(set_attr "type" "mve_move")
1039 ])
1040
1041 ;;
1042 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1043 ;;
1044 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1045   [
1046    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1047         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1048                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1049          VCMPEQQ_N))
1050   ]
1051   "TARGET_HAVE_MVE"
1052   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1053   [(set_attr "type" "mve_move")
1054 ])
1055
1056 ;;
1057 ;; [vcmpeqq_u, vcmpeqq_s])
1058 ;;
1059 (define_insn "mve_vcmpeqq_<supf><mode>"
1060   [
1061    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1062         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1063                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1064          VCMPEQQ))
1065   ]
1066   "TARGET_HAVE_MVE"
1067   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1068   [(set_attr "type" "mve_move")
1069 ])
1070
1071 ;;
1072 ;; [vcmpgeq_n_s])
1073 ;;
1074 (define_insn "mve_vcmpgeq_n_s<mode>"
1075   [
1076    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1077         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1078                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1079          VCMPGEQ_N_S))
1080   ]
1081   "TARGET_HAVE_MVE"
1082   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1083   [(set_attr "type" "mve_move")
1084 ])
1085
1086 ;;
1087 ;; [vcmpgeq_s])
1088 ;;
1089 (define_insn "mve_vcmpgeq_s<mode>"
1090   [
1091    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1092         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1093                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1094          VCMPGEQ_S))
1095   ]
1096   "TARGET_HAVE_MVE"
1097   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1098   [(set_attr "type" "mve_move")
1099 ])
1100
1101 ;;
1102 ;; [vcmpgtq_n_s])
1103 ;;
1104 (define_insn "mve_vcmpgtq_n_s<mode>"
1105   [
1106    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1107         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1108                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1109          VCMPGTQ_N_S))
1110   ]
1111   "TARGET_HAVE_MVE"
1112   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1113   [(set_attr "type" "mve_move")
1114 ])
1115
1116 ;;
1117 ;; [vcmpgtq_s])
1118 ;;
1119 (define_insn "mve_vcmpgtq_s<mode>"
1120   [
1121    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1122         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1123                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1124          VCMPGTQ_S))
1125   ]
1126   "TARGET_HAVE_MVE"
1127   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1128   [(set_attr "type" "mve_move")
1129 ])
1130
1131 ;;
1132 ;; [vcmphiq_n_u])
1133 ;;
1134 (define_insn "mve_vcmphiq_n_u<mode>"
1135   [
1136    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1137         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1138                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1139          VCMPHIQ_N_U))
1140   ]
1141   "TARGET_HAVE_MVE"
1142   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1143   [(set_attr "type" "mve_move")
1144 ])
1145
1146 ;;
1147 ;; [vcmphiq_u])
1148 ;;
1149 (define_insn "mve_vcmphiq_u<mode>"
1150   [
1151    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1152         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1153                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1154          VCMPHIQ_U))
1155   ]
1156   "TARGET_HAVE_MVE"
1157   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1158   [(set_attr "type" "mve_move")
1159 ])
1160
1161 ;;
1162 ;; [vcmpleq_n_s])
1163 ;;
1164 (define_insn "mve_vcmpleq_n_s<mode>"
1165   [
1166    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1167         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1168                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1169          VCMPLEQ_N_S))
1170   ]
1171   "TARGET_HAVE_MVE"
1172   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1173   [(set_attr "type" "mve_move")
1174 ])
1175
1176 ;;
1177 ;; [vcmpleq_s])
1178 ;;
1179 (define_insn "mve_vcmpleq_s<mode>"
1180   [
1181    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1182         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1183                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1184          VCMPLEQ_S))
1185   ]
1186   "TARGET_HAVE_MVE"
1187   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1188   [(set_attr "type" "mve_move")
1189 ])
1190
1191 ;;
1192 ;; [vcmpltq_n_s])
1193 ;;
1194 (define_insn "mve_vcmpltq_n_s<mode>"
1195   [
1196    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1197         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1198                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1199          VCMPLTQ_N_S))
1200   ]
1201   "TARGET_HAVE_MVE"
1202   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1203   [(set_attr "type" "mve_move")
1204 ])
1205
1206 ;;
1207 ;; [vcmpltq_s])
1208 ;;
1209 (define_insn "mve_vcmpltq_s<mode>"
1210   [
1211    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1212         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1213                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1214          VCMPLTQ_S))
1215   ]
1216   "TARGET_HAVE_MVE"
1217   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1218   [(set_attr "type" "mve_move")
1219 ])
1220
1221 ;;
1222 ;; [vcmpneq_n_u, vcmpneq_n_s])
1223 ;;
1224 (define_insn "mve_vcmpneq_n_<supf><mode>"
1225   [
1226    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1227         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1228                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1229          VCMPNEQ_N))
1230   ]
1231   "TARGET_HAVE_MVE"
1232   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1233   [(set_attr "type" "mve_move")
1234 ])
1235
1236 ;;
1237 ;; [veorq_u, veorq_s])
1238 ;;
1239 (define_insn "mve_veorq_u<mode>"
1240   [
1241    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1242         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1243                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1244   ]
1245   "TARGET_HAVE_MVE"
1246   "veor\t%q0, %q1, %q2"
1247   [(set_attr "type" "mve_move")
1248 ])
1249 (define_expand "mve_veorq_s<mode>"
1250   [
1251    (set (match_operand:MVE_2 0 "s_register_operand")
1252         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1253                    (match_operand:MVE_2 2 "s_register_operand")))
1254   ]
1255   "TARGET_HAVE_MVE"
1256 )
1257
1258 ;;
1259 ;; [vhaddq_n_u, vhaddq_n_s])
1260 ;;
1261 (define_insn "mve_vhaddq_n_<supf><mode>"
1262   [
1263    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1264         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1265                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1266          VHADDQ_N))
1267   ]
1268   "TARGET_HAVE_MVE"
1269   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1270   [(set_attr "type" "mve_move")
1271 ])
1272
1273 ;;
1274 ;; [vhaddq_s, vhaddq_u])
1275 ;;
1276 (define_insn "mve_vhaddq_<supf><mode>"
1277   [
1278    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1279         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1280                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1281          VHADDQ))
1282   ]
1283   "TARGET_HAVE_MVE"
1284   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1285   [(set_attr "type" "mve_move")
1286 ])
1287
1288 ;;
1289 ;; [vhcaddq_rot270_s])
1290 ;;
1291 (define_insn "mve_vhcaddq_rot270_s<mode>"
1292   [
1293    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1294         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1295                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1296          VHCADDQ_ROT270_S))
1297   ]
1298   "TARGET_HAVE_MVE"
1299   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1300   [(set_attr "type" "mve_move")
1301 ])
1302
1303 ;;
1304 ;; [vhcaddq_rot90_s])
1305 ;;
1306 (define_insn "mve_vhcaddq_rot90_s<mode>"
1307   [
1308    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1309         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1310                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1311          VHCADDQ_ROT90_S))
1312   ]
1313   "TARGET_HAVE_MVE"
1314   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1315   [(set_attr "type" "mve_move")
1316 ])
1317
1318 ;;
1319 ;; [vhsubq_n_u, vhsubq_n_s])
1320 ;;
1321 (define_insn "mve_vhsubq_n_<supf><mode>"
1322   [
1323    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1324         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1325                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1326          VHSUBQ_N))
1327   ]
1328   "TARGET_HAVE_MVE"
1329   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1330   [(set_attr "type" "mve_move")
1331 ])
1332
1333 ;;
1334 ;; [vhsubq_s, vhsubq_u])
1335 ;;
1336 (define_insn "mve_vhsubq_<supf><mode>"
1337   [
1338    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1339         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1340                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1341          VHSUBQ))
1342   ]
1343   "TARGET_HAVE_MVE"
1344   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1345   [(set_attr "type" "mve_move")
1346 ])
1347
1348 ;;
1349 ;; [vmaxaq_s])
1350 ;;
1351 (define_insn "mve_vmaxaq_s<mode>"
1352   [
1353    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1354         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1355                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1356          VMAXAQ_S))
1357   ]
1358   "TARGET_HAVE_MVE"
1359   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1360   [(set_attr "type" "mve_move")
1361 ])
1362
1363 ;;
1364 ;; [vmaxavq_s])
1365 ;;
1366 (define_insn "mve_vmaxavq_s<mode>"
1367   [
1368    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1369         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1370                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1371          VMAXAVQ_S))
1372   ]
1373   "TARGET_HAVE_MVE"
1374   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1375   [(set_attr "type" "mve_move")
1376 ])
1377
1378 ;;
1379 ;; [vmaxq_u, vmaxq_s])
1380 ;;
1381 (define_insn "mve_vmaxq_s<mode>"
1382   [
1383    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1384         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1385                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1386   ]
1387   "TARGET_HAVE_MVE"
1388   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1389   [(set_attr "type" "mve_move")
1390 ])
1391
1392 (define_insn "mve_vmaxq_u<mode>"
1393   [
1394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1395         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1396                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1397   ]
1398   "TARGET_HAVE_MVE"
1399   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1400   [(set_attr "type" "mve_move")
1401 ])
1402
1403 ;;
1404 ;; [vmaxvq_u, vmaxvq_s])
1405 ;;
1406 (define_insn "mve_vmaxvq_<supf><mode>"
1407   [
1408    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1409         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1410                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1411          VMAXVQ))
1412   ]
1413   "TARGET_HAVE_MVE"
1414   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1415   [(set_attr "type" "mve_move")
1416 ])
1417
1418 ;;
1419 ;; [vminaq_s])
1420 ;;
1421 (define_insn "mve_vminaq_s<mode>"
1422   [
1423    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1424         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1425                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1426          VMINAQ_S))
1427   ]
1428   "TARGET_HAVE_MVE"
1429   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1430   [(set_attr "type" "mve_move")
1431 ])
1432
1433 ;;
1434 ;; [vminavq_s])
1435 ;;
1436 (define_insn "mve_vminavq_s<mode>"
1437   [
1438    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1439         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1440                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1441          VMINAVQ_S))
1442   ]
1443   "TARGET_HAVE_MVE"
1444   "vminav.s%#<V_sz_elem>\t%0, %q2"
1445   [(set_attr "type" "mve_move")
1446 ])
1447
1448 ;;
1449 ;; [vminq_s, vminq_u])
1450 ;;
1451 (define_insn "mve_vminq_s<mode>"
1452   [
1453    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1454         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1455                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1456   ]
1457   "TARGET_HAVE_MVE"
1458   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1459   [(set_attr "type" "mve_move")
1460 ])
1461
1462 (define_insn "mve_vminq_u<mode>"
1463   [
1464    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1465         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1466                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1467   ]
1468   "TARGET_HAVE_MVE"
1469   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1470   [(set_attr "type" "mve_move")
1471 ])
1472
1473 ;;
1474 ;; [vminvq_u, vminvq_s])
1475 ;;
1476 (define_insn "mve_vminvq_<supf><mode>"
1477   [
1478    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1479         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1480                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1481          VMINVQ))
1482   ]
1483   "TARGET_HAVE_MVE"
1484   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1485   [(set_attr "type" "mve_move")
1486 ])
1487
1488 ;;
1489 ;; [vmladavq_u, vmladavq_s])
1490 ;;
1491 (define_insn "mve_vmladavq_<supf><mode>"
1492   [
1493    (set (match_operand:SI 0 "s_register_operand" "=Te")
1494         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1495                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1496          VMLADAVQ))
1497   ]
1498   "TARGET_HAVE_MVE"
1499   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1500   [(set_attr "type" "mve_move")
1501 ])
1502
1503 ;;
1504 ;; [vmladavxq_s])
1505 ;;
1506 (define_insn "mve_vmladavxq_s<mode>"
1507   [
1508    (set (match_operand:SI 0 "s_register_operand" "=Te")
1509         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1510                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1511          VMLADAVXQ_S))
1512   ]
1513   "TARGET_HAVE_MVE"
1514   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1515   [(set_attr "type" "mve_move")
1516 ])
1517
1518 ;;
1519 ;; [vmlsdavq_s])
1520 ;;
1521 (define_insn "mve_vmlsdavq_s<mode>"
1522   [
1523    (set (match_operand:SI 0 "s_register_operand" "=Te")
1524         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1525                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1526          VMLSDAVQ_S))
1527   ]
1528   "TARGET_HAVE_MVE"
1529   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1530   [(set_attr "type" "mve_move")
1531 ])
1532
1533 ;;
1534 ;; [vmlsdavxq_s])
1535 ;;
1536 (define_insn "mve_vmlsdavxq_s<mode>"
1537   [
1538    (set (match_operand:SI 0 "s_register_operand" "=Te")
1539         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1540                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1541          VMLSDAVXQ_S))
1542   ]
1543   "TARGET_HAVE_MVE"
1544   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1545   [(set_attr "type" "mve_move")
1546 ])
1547
1548 ;;
1549 ;; [vmulhq_s, vmulhq_u])
1550 ;;
1551 (define_insn "mve_vmulhq_<supf><mode>"
1552   [
1553    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1554         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1555                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1556          VMULHQ))
1557   ]
1558   "TARGET_HAVE_MVE"
1559   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1560   [(set_attr "type" "mve_move")
1561 ])
1562
1563 ;;
1564 ;; [vmullbq_int_u, vmullbq_int_s])
1565 ;;
1566 (define_insn "mve_vmullbq_int_<supf><mode>"
1567   [
1568    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1569         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1570                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1571          VMULLBQ_INT))
1572   ]
1573   "TARGET_HAVE_MVE"
1574   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1575   [(set_attr "type" "mve_move")
1576 ])
1577
1578 ;;
1579 ;; [vmulltq_int_u, vmulltq_int_s])
1580 ;;
1581 (define_insn "mve_vmulltq_int_<supf><mode>"
1582   [
1583    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1584         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1585                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1586          VMULLTQ_INT))
1587   ]
1588   "TARGET_HAVE_MVE"
1589   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1590   [(set_attr "type" "mve_move")
1591 ])
1592
1593 ;;
1594 ;; [vmulq_n_u, vmulq_n_s])
1595 ;;
1596 (define_insn "mve_vmulq_n_<supf><mode>"
1597   [
1598    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1599         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1600                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1601          VMULQ_N))
1602   ]
1603   "TARGET_HAVE_MVE"
1604   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1605   [(set_attr "type" "mve_move")
1606 ])
1607
1608 ;;
1609 ;; [vmulq_u, vmulq_s])
1610 ;;
1611 (define_insn "mve_vmulq_<supf><mode>"
1612   [
1613    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1614         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1615                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1616          VMULQ))
1617   ]
1618   "TARGET_HAVE_MVE"
1619   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1620   [(set_attr "type" "mve_move")
1621 ])
1622
1623 (define_insn "mve_vmulq<mode>"
1624   [
1625    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1626         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1627                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1628   ]
1629   "TARGET_HAVE_MVE"
1630   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1631   [(set_attr "type" "mve_move")
1632 ])
1633
1634 ;;
1635 ;; [vornq_u, vornq_s])
1636 ;;
1637 (define_insn "mve_vornq_<supf><mode>"
1638   [
1639    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1640         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1641                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1642          VORNQ))
1643   ]
1644   "TARGET_HAVE_MVE"
1645   "vorn %q0, %q1, %q2"
1646   [(set_attr "type" "mve_move")
1647 ])
1648
1649 ;;
1650 ;; [vorrq_s, vorrq_u])
1651 ;;
1652 ;; signed and unsigned versions are the same: define the unsigned
1653 ;; insn, and use an expander for the signed one as we still reference
1654 ;; both names from arm_mve.h.
1655 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1656 (define_insn "mve_vorrq_s<mode>"
1657   [
1658    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1659         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1660                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1661   ]
1662   "TARGET_HAVE_MVE"
1663   "@
1664    vorr\t%q0, %q1, %q2
1665    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1666   [(set_attr "type" "mve_move")
1667 ])
1668 (define_expand "mve_vorrq_u<mode>"
1669   [
1670    (set (match_operand:MVE_2 0 "s_register_operand")
1671         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1672                    (match_operand:MVE_2 2 "neon_logic_op2")))
1673   ]
1674   "TARGET_HAVE_MVE"
1675 )
1676
1677 ;;
1678 ;; [vqaddq_n_s, vqaddq_n_u])
1679 ;;
1680 (define_insn "mve_vqaddq_n_<supf><mode>"
1681   [
1682    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1683         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1684                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1685          VQADDQ_N))
1686   ]
1687   "TARGET_HAVE_MVE"
1688   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1689   [(set_attr "type" "mve_move")
1690 ])
1691
1692 ;;
1693 ;; [vqaddq_u, vqaddq_s])
1694 ;;
1695 (define_insn "mve_vqaddq_<supf><mode>"
1696   [
1697    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1698         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1699                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1700          VQADDQ))
1701   ]
1702   "TARGET_HAVE_MVE"
1703   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1704   [(set_attr "type" "mve_move")
1705 ])
1706
1707 ;;
1708 ;; [vqdmulhq_n_s])
1709 ;;
1710 (define_insn "mve_vqdmulhq_n_s<mode>"
1711   [
1712    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1713         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1714                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1715          VQDMULHQ_N_S))
1716   ]
1717   "TARGET_HAVE_MVE"
1718   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1719   [(set_attr "type" "mve_move")
1720 ])
1721
1722 ;;
1723 ;; [vqdmulhq_s])
1724 ;;
1725 (define_insn "mve_vqdmulhq_s<mode>"
1726   [
1727    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1728         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1729                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1730          VQDMULHQ_S))
1731   ]
1732   "TARGET_HAVE_MVE"
1733   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1734   [(set_attr "type" "mve_move")
1735 ])
1736
1737 ;;
1738 ;; [vqrdmulhq_n_s])
1739 ;;
1740 (define_insn "mve_vqrdmulhq_n_s<mode>"
1741   [
1742    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1743         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1744                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1745          VQRDMULHQ_N_S))
1746   ]
1747   "TARGET_HAVE_MVE"
1748   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1749   [(set_attr "type" "mve_move")
1750 ])
1751
1752 ;;
1753 ;; [vqrdmulhq_s])
1754 ;;
1755 (define_insn "mve_vqrdmulhq_s<mode>"
1756   [
1757    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1758         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1759                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1760          VQRDMULHQ_S))
1761   ]
1762   "TARGET_HAVE_MVE"
1763   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1764   [(set_attr "type" "mve_move")
1765 ])
1766
1767 ;;
1768 ;; [vqrshlq_n_s, vqrshlq_n_u])
1769 ;;
1770 (define_insn "mve_vqrshlq_n_<supf><mode>"
1771   [
1772    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1773         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1774                        (match_operand:SI 2 "s_register_operand" "r")]
1775          VQRSHLQ_N))
1776   ]
1777   "TARGET_HAVE_MVE"
1778   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1779   [(set_attr "type" "mve_move")
1780 ])
1781
1782 ;;
1783 ;; [vqrshlq_s, vqrshlq_u])
1784 ;;
1785 (define_insn "mve_vqrshlq_<supf><mode>"
1786   [
1787    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1788         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1789                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1790          VQRSHLQ))
1791   ]
1792   "TARGET_HAVE_MVE"
1793   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1794   [(set_attr "type" "mve_move")
1795 ])
1796
1797 ;;
1798 ;; [vqshlq_n_s, vqshlq_n_u])
1799 ;;
1800 (define_insn "mve_vqshlq_n_<supf><mode>"
1801   [
1802    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1803         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1804                        (match_operand:SI 2 "immediate_operand" "i")]
1805          VQSHLQ_N))
1806   ]
1807   "TARGET_HAVE_MVE"
1808   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1809   [(set_attr "type" "mve_move")
1810 ])
1811
1812 ;;
1813 ;; [vqshlq_r_u, vqshlq_r_s])
1814 ;;
1815 (define_insn "mve_vqshlq_r_<supf><mode>"
1816   [
1817    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1818         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1819                        (match_operand:SI 2 "s_register_operand" "r")]
1820          VQSHLQ_R))
1821   ]
1822   "TARGET_HAVE_MVE"
1823   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1824   [(set_attr "type" "mve_move")
1825 ])
1826
1827 ;;
1828 ;; [vqshlq_s, vqshlq_u])
1829 ;;
1830 (define_insn "mve_vqshlq_<supf><mode>"
1831   [
1832    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1833         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1834                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1835          VQSHLQ))
1836   ]
1837   "TARGET_HAVE_MVE"
1838   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1839   [(set_attr "type" "mve_move")
1840 ])
1841
1842 ;;
1843 ;; [vqshluq_n_s])
1844 ;;
1845 (define_insn "mve_vqshluq_n_s<mode>"
1846   [
1847    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1848         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1849                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1850          VQSHLUQ_N_S))
1851   ]
1852   "TARGET_HAVE_MVE"
1853   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1854   [(set_attr "type" "mve_move")
1855 ])
1856
1857 ;;
1858 ;; [vqsubq_n_s, vqsubq_n_u])
1859 ;;
1860 (define_insn "mve_vqsubq_n_<supf><mode>"
1861   [
1862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1864                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1865          VQSUBQ_N))
1866   ]
1867   "TARGET_HAVE_MVE"
1868   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1869   [(set_attr "type" "mve_move")
1870 ])
1871
1872 ;;
1873 ;; [vqsubq_u, vqsubq_s])
1874 ;;
1875 (define_insn "mve_vqsubq_<supf><mode>"
1876   [
1877    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1878         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1879                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1880          VQSUBQ))
1881   ]
1882   "TARGET_HAVE_MVE"
1883   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1884   [(set_attr "type" "mve_move")
1885 ])
1886
1887 ;;
1888 ;; [vrhaddq_s, vrhaddq_u])
1889 ;;
1890 (define_insn "mve_vrhaddq_<supf><mode>"
1891   [
1892    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1893         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1894                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1895          VRHADDQ))
1896   ]
1897   "TARGET_HAVE_MVE"
1898   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1899   [(set_attr "type" "mve_move")
1900 ])
1901
1902 ;;
1903 ;; [vrmulhq_s, vrmulhq_u])
1904 ;;
1905 (define_insn "mve_vrmulhq_<supf><mode>"
1906   [
1907    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1908         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1909                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1910          VRMULHQ))
1911   ]
1912   "TARGET_HAVE_MVE"
1913   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1914   [(set_attr "type" "mve_move")
1915 ])
1916
1917 ;;
1918 ;; [vrshlq_n_u, vrshlq_n_s])
1919 ;;
1920 (define_insn "mve_vrshlq_n_<supf><mode>"
1921   [
1922    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1923         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1924                        (match_operand:SI 2 "s_register_operand" "r")]
1925          VRSHLQ_N))
1926   ]
1927   "TARGET_HAVE_MVE"
1928   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1929   [(set_attr "type" "mve_move")
1930 ])
1931
1932 ;;
1933 ;; [vrshlq_s, vrshlq_u])
1934 ;;
1935 (define_insn "mve_vrshlq_<supf><mode>"
1936   [
1937    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1938         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1939                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1940          VRSHLQ))
1941   ]
1942   "TARGET_HAVE_MVE"
1943   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1944   [(set_attr "type" "mve_move")
1945 ])
1946
1947 ;;
1948 ;; [vrshrq_n_s, vrshrq_n_u])
1949 ;;
1950 (define_insn "mve_vrshrq_n_<supf><mode>"
1951   [
1952    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1953         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1954                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1955          VRSHRQ_N))
1956   ]
1957   "TARGET_HAVE_MVE"
1958   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1959   [(set_attr "type" "mve_move")
1960 ])
1961
1962 ;;
1963 ;; [vshlq_n_u, vshlq_n_s])
1964 ;;
1965 (define_insn "mve_vshlq_n_<supf><mode>"
1966   [
1967    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1968         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1969                        (match_operand:SI 2 "immediate_operand" "i")]
1970          VSHLQ_N))
1971   ]
1972   "TARGET_HAVE_MVE"
1973   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1974   [(set_attr "type" "mve_move")
1975 ])
1976
1977 ;;
1978 ;; [vshlq_r_s, vshlq_r_u])
1979 ;;
1980 (define_insn "mve_vshlq_r_<supf><mode>"
1981   [
1982    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1983         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1984                        (match_operand:SI 2 "s_register_operand" "r")]
1985          VSHLQ_R))
1986   ]
1987   "TARGET_HAVE_MVE"
1988   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1989   [(set_attr "type" "mve_move")
1990 ])
1991
1992 ;;
1993 ;; [vsubq_n_s, vsubq_n_u])
1994 ;;
1995 (define_insn "mve_vsubq_n_<supf><mode>"
1996   [
1997    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1998         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1999                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2000          VSUBQ_N))
2001   ]
2002   "TARGET_HAVE_MVE"
2003   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
2004   [(set_attr "type" "mve_move")
2005 ])
2006
2007 ;;
2008 ;; [vsubq_s, vsubq_u])
2009 ;;
2010 (define_insn "mve_vsubq_<supf><mode>"
2011   [
2012    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2013         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2014                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2015          VSUBQ))
2016   ]
2017   "TARGET_HAVE_MVE"
2018   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2019   [(set_attr "type" "mve_move")
2020 ])
2021
2022 (define_insn "mve_vsubq<mode>"
2023   [
2024    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2025         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
2026                      (match_operand:MVE_2 2 "s_register_operand" "w")))
2027   ]
2028   "TARGET_HAVE_MVE"
2029   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2030   [(set_attr "type" "mve_move")
2031 ])
2032
2033 ;;
2034 ;; [vabdq_f])
2035 ;;
2036 (define_insn "mve_vabdq_f<mode>"
2037   [
2038    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2039         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2040                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2041          VABDQ_F))
2042   ]
2043   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2044   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2045   [(set_attr "type" "mve_move")
2046 ])
2047
2048 ;;
2049 ;; [vaddlvaq_s vaddlvaq_u])
2050 ;;
2051 (define_insn "mve_vaddlvaq_<supf>v4si"
2052   [
2053    (set (match_operand:DI 0 "s_register_operand" "=r")
2054         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2055                     (match_operand:V4SI 2 "s_register_operand" "w")]
2056          VADDLVAQ))
2057   ]
2058   "TARGET_HAVE_MVE"
2059   "vaddlva.<supf>32 %Q0, %R0, %q2"
2060   [(set_attr "type" "mve_move")
2061 ])
2062
2063 ;;
2064 ;; [vaddq_n_f])
2065 ;;
2066 (define_insn "mve_vaddq_n_f<mode>"
2067   [
2068    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2069         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2070                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2071          VADDQ_N_F))
2072   ]
2073   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2074   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2075   [(set_attr "type" "mve_move")
2076 ])
2077
2078 ;;
2079 ;; [vandq_f])
2080 ;;
2081 (define_insn "mve_vandq_f<mode>"
2082   [
2083    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2084         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2085                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2086   ]
2087   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2088   "vand %q0, %q1, %q2"
2089   [(set_attr "type" "mve_move")
2090 ])
2091
2092 ;;
2093 ;; [vbicq_f])
2094 ;;
2095 (define_insn "mve_vbicq_f<mode>"
2096   [
2097    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2098         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
2099                               (match_operand:MVE_0 2 "s_register_operand" "w")))
2100   ]
2101   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2102   "vbic %q0, %q1, %q2"
2103   [(set_attr "type" "mve_move")
2104 ])
2105
2106 ;;
2107 ;; [vbicq_n_s, vbicq_n_u])
2108 ;;
2109 (define_insn "mve_vbicq_n_<supf><mode>"
2110   [
2111    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2112         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2113                        (match_operand:SI 2 "immediate_operand" "i")]
2114          VBICQ_N))
2115   ]
2116   "TARGET_HAVE_MVE"
2117   "vbic.i%#<V_sz_elem>  %q0, %2"
2118   [(set_attr "type" "mve_move")
2119 ])
2120
2121 ;;
2122 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
2123 ;;
2124 (define_insn "mve_vcaddq<mve_rot><mode>"
2125   [
2126    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2127         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2128                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2129          VCADD))
2130   ]
2131   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2132   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
2133   [(set_attr "type" "mve_move")
2134 ])
2135
2136 ;;
2137 ;; [vcmpeqq_f])
2138 ;;
2139 (define_insn "mve_vcmpeqq_f<mode>"
2140   [
2141    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2142         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2143                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2144          VCMPEQQ_F))
2145   ]
2146   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2147   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2148   [(set_attr "type" "mve_move")
2149 ])
2150
2151 ;;
2152 ;; [vcmpeqq_n_f])
2153 ;;
2154 (define_insn "mve_vcmpeqq_n_f<mode>"
2155   [
2156    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2157         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2158                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2159          VCMPEQQ_N_F))
2160   ]
2161   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2162   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2163   [(set_attr "type" "mve_move")
2164 ])
2165
2166 ;;
2167 ;; [vcmpgeq_f])
2168 ;;
2169 (define_insn "mve_vcmpgeq_f<mode>"
2170   [
2171    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2172         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2173                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2174          VCMPGEQ_F))
2175   ]
2176   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2177   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2178   [(set_attr "type" "mve_move")
2179 ])
2180
2181 ;;
2182 ;; [vcmpgeq_n_f])
2183 ;;
2184 (define_insn "mve_vcmpgeq_n_f<mode>"
2185   [
2186    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2187         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2188                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2189          VCMPGEQ_N_F))
2190   ]
2191   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2192   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2193   [(set_attr "type" "mve_move")
2194 ])
2195
2196 ;;
2197 ;; [vcmpgtq_f])
2198 ;;
2199 (define_insn "mve_vcmpgtq_f<mode>"
2200   [
2201    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2202         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2203                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2204          VCMPGTQ_F))
2205   ]
2206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2207   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2208   [(set_attr "type" "mve_move")
2209 ])
2210
2211 ;;
2212 ;; [vcmpgtq_n_f])
2213 ;;
2214 (define_insn "mve_vcmpgtq_n_f<mode>"
2215   [
2216    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2217         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2218                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2219          VCMPGTQ_N_F))
2220   ]
2221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2222   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2223   [(set_attr "type" "mve_move")
2224 ])
2225
2226 ;;
2227 ;; [vcmpleq_f])
2228 ;;
2229 (define_insn "mve_vcmpleq_f<mode>"
2230   [
2231    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2232         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2233                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2234          VCMPLEQ_F))
2235   ]
2236   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2237   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2238   [(set_attr "type" "mve_move")
2239 ])
2240
2241 ;;
2242 ;; [vcmpleq_n_f])
2243 ;;
2244 (define_insn "mve_vcmpleq_n_f<mode>"
2245   [
2246    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2247         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2248                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2249          VCMPLEQ_N_F))
2250   ]
2251   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2252   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2253   [(set_attr "type" "mve_move")
2254 ])
2255
2256 ;;
2257 ;; [vcmpltq_f])
2258 ;;
2259 (define_insn "mve_vcmpltq_f<mode>"
2260   [
2261    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2262         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2263                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2264          VCMPLTQ_F))
2265   ]
2266   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2267   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2268   [(set_attr "type" "mve_move")
2269 ])
2270
2271 ;;
2272 ;; [vcmpltq_n_f])
2273 ;;
2274 (define_insn "mve_vcmpltq_n_f<mode>"
2275   [
2276    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2277         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2278                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2279          VCMPLTQ_N_F))
2280   ]
2281   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2282   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2283   [(set_attr "type" "mve_move")
2284 ])
2285
2286 ;;
2287 ;; [vcmpneq_f])
2288 ;;
2289 (define_insn "mve_vcmpneq_f<mode>"
2290   [
2291    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2292         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2293                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2294          VCMPNEQ_F))
2295   ]
2296   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2297   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2298   [(set_attr "type" "mve_move")
2299 ])
2300
2301 ;;
2302 ;; [vcmpneq_n_f])
2303 ;;
2304 (define_insn "mve_vcmpneq_n_f<mode>"
2305   [
2306    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2307         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2308                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2309          VCMPNEQ_N_F))
2310   ]
2311   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2312   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2313   [(set_attr "type" "mve_move")
2314 ])
2315
2316 ;;
2317 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
2318 ;;
2319 (define_insn "mve_vcmulq<mve_rot><mode>"
2320   [
2321    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2322         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2323                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2324          VCMUL))
2325   ]
2326   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2327   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
2328   [(set_attr "type" "mve_move")
2329 ])
2330
2331 ;;
2332 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2333 ;;
2334 (define_insn "mve_vctp<mode1>q_mhi"
2335   [
2336    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2337         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2338                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2339          VCTPQ_M))
2340   ]
2341   "TARGET_HAVE_MVE"
2342   "vpst\;vctpt.<mode1> %1"
2343   [(set_attr "type" "mve_move")
2344    (set_attr "length""8")])
2345
2346 ;;
2347 ;; [vcvtbq_f16_f32])
2348 ;;
2349 (define_insn "mve_vcvtbq_f16_f32v8hf"
2350   [
2351    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2352         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2353                       (match_operand:V4SF 2 "s_register_operand" "w")]
2354          VCVTBQ_F16_F32))
2355   ]
2356   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2357   "vcvtb.f16.f32 %q0, %q2"
2358   [(set_attr "type" "mve_move")
2359 ])
2360
2361 ;;
2362 ;; [vcvttq_f16_f32])
2363 ;;
2364 (define_insn "mve_vcvttq_f16_f32v8hf"
2365   [
2366    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2367         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2368                       (match_operand:V4SF 2 "s_register_operand" "w")]
2369          VCVTTQ_F16_F32))
2370   ]
2371   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2372   "vcvtt.f16.f32 %q0, %q2"
2373   [(set_attr "type" "mve_move")
2374 ])
2375
2376 ;;
2377 ;; [veorq_f])
2378 ;;
2379 (define_insn "mve_veorq_f<mode>"
2380   [
2381    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2382         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2383                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2384   ]
2385   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2386   "veor %q0, %q1, %q2"
2387   [(set_attr "type" "mve_move")
2388 ])
2389
2390 ;;
2391 ;; [vmaxnmaq_f])
2392 ;;
2393 (define_insn "mve_vmaxnmaq_f<mode>"
2394   [
2395    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2396         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2397                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2398          VMAXNMAQ_F))
2399   ]
2400   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2401   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2402   [(set_attr "type" "mve_move")
2403 ])
2404
2405 ;;
2406 ;; [vmaxnmavq_f])
2407 ;;
2408 (define_insn "mve_vmaxnmavq_f<mode>"
2409   [
2410    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2411         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2412                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2413          VMAXNMAVQ_F))
2414   ]
2415   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2416   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2417   [(set_attr "type" "mve_move")
2418 ])
2419
2420 ;;
2421 ;; [vmaxnmq_f])
2422 ;;
2423 (define_insn "mve_vmaxnmq_f<mode>"
2424   [
2425    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2426         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2427                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2428   ]
2429   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2430   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2431   [(set_attr "type" "mve_move")
2432 ])
2433
2434 ;;
2435 ;; [vmaxnmvq_f])
2436 ;;
2437 (define_insn "mve_vmaxnmvq_f<mode>"
2438   [
2439    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2440         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2441                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2442          VMAXNMVQ_F))
2443   ]
2444   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2445   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2446   [(set_attr "type" "mve_move")
2447 ])
2448
2449 ;;
2450 ;; [vminnmaq_f])
2451 ;;
2452 (define_insn "mve_vminnmaq_f<mode>"
2453   [
2454    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2455         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2456                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2457          VMINNMAQ_F))
2458   ]
2459   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2460   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2461   [(set_attr "type" "mve_move")
2462 ])
2463
2464 ;;
2465 ;; [vminnmavq_f])
2466 ;;
2467 (define_insn "mve_vminnmavq_f<mode>"
2468   [
2469    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2470         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2471                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2472          VMINNMAVQ_F))
2473   ]
2474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2475   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2476   [(set_attr "type" "mve_move")
2477 ])
2478
2479 ;;
2480 ;; [vminnmq_f])
2481 ;;
2482 (define_insn "mve_vminnmq_f<mode>"
2483   [
2484    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2485         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2486                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2487   ]
2488   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2489   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2490   [(set_attr "type" "mve_move")
2491 ])
2492
2493 ;;
2494 ;; [vminnmvq_f])
2495 ;;
2496 (define_insn "mve_vminnmvq_f<mode>"
2497   [
2498    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2499         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2500                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2501          VMINNMVQ_F))
2502   ]
2503   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2504   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2505   [(set_attr "type" "mve_move")
2506 ])
2507
2508 ;;
2509 ;; [vmlaldavq_u, vmlaldavq_s])
2510 ;;
2511 (define_insn "mve_vmlaldavq_<supf><mode>"
2512   [
2513    (set (match_operand:DI 0 "s_register_operand" "=r")
2514         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2515                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2516          VMLALDAVQ))
2517   ]
2518   "TARGET_HAVE_MVE"
2519   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2520   [(set_attr "type" "mve_move")
2521 ])
2522
2523 ;;
2524 ;; [vmlaldavxq_s])
2525 ;;
2526 (define_insn "mve_vmlaldavxq_s<mode>"
2527   [
2528    (set (match_operand:DI 0 "s_register_operand" "=r")
2529         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2530                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2531          VMLALDAVXQ_S))
2532   ]
2533   "TARGET_HAVE_MVE"
2534   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2535   [(set_attr "type" "mve_move")
2536 ])
2537
2538 ;;
2539 ;; [vmlsldavq_s])
2540 ;;
2541 (define_insn "mve_vmlsldavq_s<mode>"
2542   [
2543    (set (match_operand:DI 0 "s_register_operand" "=r")
2544         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2545                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2546          VMLSLDAVQ_S))
2547   ]
2548   "TARGET_HAVE_MVE"
2549   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2550   [(set_attr "type" "mve_move")
2551 ])
2552
2553 ;;
2554 ;; [vmlsldavxq_s])
2555 ;;
2556 (define_insn "mve_vmlsldavxq_s<mode>"
2557   [
2558    (set (match_operand:DI 0 "s_register_operand" "=r")
2559         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2560                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2561          VMLSLDAVXQ_S))
2562   ]
2563   "TARGET_HAVE_MVE"
2564   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2565   [(set_attr "type" "mve_move")
2566 ])
2567
2568 ;;
2569 ;; [vmovnbq_u, vmovnbq_s])
2570 ;;
2571 (define_insn "mve_vmovnbq_<supf><mode>"
2572   [
2573    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2574         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2575                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2576          VMOVNBQ))
2577   ]
2578   "TARGET_HAVE_MVE"
2579   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2580   [(set_attr "type" "mve_move")
2581 ])
2582
2583 ;;
2584 ;; [vmovntq_s, vmovntq_u])
2585 ;;
2586 (define_insn "mve_vmovntq_<supf><mode>"
2587   [
2588    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2589         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2590                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2591          VMOVNTQ))
2592   ]
2593   "TARGET_HAVE_MVE"
2594   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2595   [(set_attr "type" "mve_move")
2596 ])
2597
2598 ;;
2599 ;; [vmulq_f])
2600 ;;
2601 (define_insn "mve_vmulq_f<mode>"
2602   [
2603    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2604         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2605                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2606   ]
2607   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2608   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2609   [(set_attr "type" "mve_move")
2610 ])
2611
2612 ;;
2613 ;; [vmulq_n_f])
2614 ;;
2615 (define_insn "mve_vmulq_n_f<mode>"
2616   [
2617    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2618         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2619                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2620          VMULQ_N_F))
2621   ]
2622   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2623   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2624   [(set_attr "type" "mve_move")
2625 ])
2626
2627 ;;
2628 ;; [vornq_f])
2629 ;;
2630 (define_insn "mve_vornq_f<mode>"
2631   [
2632    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2633         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2634                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2635          VORNQ_F))
2636   ]
2637   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2638   "vorn %q0, %q1, %q2"
2639   [(set_attr "type" "mve_move")
2640 ])
2641
2642 ;;
2643 ;; [vorrq_f])
2644 ;;
2645 (define_insn "mve_vorrq_f<mode>"
2646   [
2647    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2648         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2649                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2650   ]
2651   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2652   "vorr %q0, %q1, %q2"
2653   [(set_attr "type" "mve_move")
2654 ])
2655
2656 ;;
2657 ;; [vorrq_n_u, vorrq_n_s])
2658 ;;
2659 (define_insn "mve_vorrq_n_<supf><mode>"
2660   [
2661    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2662         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2663                        (match_operand:SI 2 "immediate_operand" "i")]
2664          VORRQ_N))
2665   ]
2666   "TARGET_HAVE_MVE"
2667   "vorr.i%#<V_sz_elem>  %q0, %2"
2668   [(set_attr "type" "mve_move")
2669 ])
2670
2671 ;;
2672 ;; [vqdmullbq_n_s])
2673 ;;
2674 (define_insn "mve_vqdmullbq_n_s<mode>"
2675   [
2676    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2677         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2678                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2679          VQDMULLBQ_N_S))
2680   ]
2681   "TARGET_HAVE_MVE"
2682   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2683   [(set_attr "type" "mve_move")
2684 ])
2685
2686 ;;
2687 ;; [vqdmullbq_s])
2688 ;;
2689 (define_insn "mve_vqdmullbq_s<mode>"
2690   [
2691    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2692         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2693                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2694          VQDMULLBQ_S))
2695   ]
2696   "TARGET_HAVE_MVE"
2697   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2698   [(set_attr "type" "mve_move")
2699 ])
2700
2701 ;;
2702 ;; [vqdmulltq_n_s])
2703 ;;
2704 (define_insn "mve_vqdmulltq_n_s<mode>"
2705   [
2706    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2707         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2708                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2709          VQDMULLTQ_N_S))
2710   ]
2711   "TARGET_HAVE_MVE"
2712   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2713   [(set_attr "type" "mve_move")
2714 ])
2715
2716 ;;
2717 ;; [vqdmulltq_s])
2718 ;;
2719 (define_insn "mve_vqdmulltq_s<mode>"
2720   [
2721    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2722         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2723                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2724          VQDMULLTQ_S))
2725   ]
2726   "TARGET_HAVE_MVE"
2727   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2728   [(set_attr "type" "mve_move")
2729 ])
2730
2731 ;;
2732 ;; [vqmovnbq_u, vqmovnbq_s])
2733 ;;
2734 (define_insn "mve_vqmovnbq_<supf><mode>"
2735   [
2736    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2737         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2738                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2739          VQMOVNBQ))
2740   ]
2741   "TARGET_HAVE_MVE"
2742   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2743   [(set_attr "type" "mve_move")
2744 ])
2745
2746 ;;
2747 ;; [vqmovntq_u, vqmovntq_s])
2748 ;;
2749 (define_insn "mve_vqmovntq_<supf><mode>"
2750   [
2751    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2752         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2753                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2754          VQMOVNTQ))
2755   ]
2756   "TARGET_HAVE_MVE"
2757   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2758   [(set_attr "type" "mve_move")
2759 ])
2760
2761 ;;
2762 ;; [vqmovunbq_s])
2763 ;;
2764 (define_insn "mve_vqmovunbq_s<mode>"
2765   [
2766    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2767         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2768                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2769          VQMOVUNBQ_S))
2770   ]
2771   "TARGET_HAVE_MVE"
2772   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2773   [(set_attr "type" "mve_move")
2774 ])
2775
2776 ;;
2777 ;; [vqmovuntq_s])
2778 ;;
2779 (define_insn "mve_vqmovuntq_s<mode>"
2780   [
2781    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2782         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2783                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2784          VQMOVUNTQ_S))
2785   ]
2786   "TARGET_HAVE_MVE"
2787   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2788   [(set_attr "type" "mve_move")
2789 ])
2790
2791 ;;
2792 ;; [vrmlaldavhxq_s])
2793 ;;
2794 (define_insn "mve_vrmlaldavhxq_sv4si"
2795   [
2796    (set (match_operand:DI 0 "s_register_operand" "=r")
2797         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2798                     (match_operand:V4SI 2 "s_register_operand" "w")]
2799          VRMLALDAVHXQ_S))
2800   ]
2801   "TARGET_HAVE_MVE"
2802   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2803   [(set_attr "type" "mve_move")
2804 ])
2805
2806 ;;
2807 ;; [vrmlsldavhq_s])
2808 ;;
2809 (define_insn "mve_vrmlsldavhq_sv4si"
2810   [
2811    (set (match_operand:DI 0 "s_register_operand" "=r")
2812         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2813                     (match_operand:V4SI 2 "s_register_operand" "w")]
2814          VRMLSLDAVHQ_S))
2815   ]
2816   "TARGET_HAVE_MVE"
2817   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2818   [(set_attr "type" "mve_move")
2819 ])
2820
2821 ;;
2822 ;; [vrmlsldavhxq_s])
2823 ;;
2824 (define_insn "mve_vrmlsldavhxq_sv4si"
2825   [
2826    (set (match_operand:DI 0 "s_register_operand" "=r")
2827         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2828                     (match_operand:V4SI 2 "s_register_operand" "w")]
2829          VRMLSLDAVHXQ_S))
2830   ]
2831   "TARGET_HAVE_MVE"
2832   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2833   [(set_attr "type" "mve_move")
2834 ])
2835
2836 ;;
2837 ;; [vshllbq_n_s, vshllbq_n_u])
2838 ;;
2839 (define_insn "mve_vshllbq_n_<supf><mode>"
2840   [
2841    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2842         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2843                                   (match_operand:SI 2 "immediate_operand" "i")]
2844          VSHLLBQ_N))
2845   ]
2846   "TARGET_HAVE_MVE"
2847   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2848   [(set_attr "type" "mve_move")
2849 ])
2850
2851 ;;
2852 ;; [vshlltq_n_u, vshlltq_n_s])
2853 ;;
2854 (define_insn "mve_vshlltq_n_<supf><mode>"
2855   [
2856    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2857         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2858                                   (match_operand:SI 2 "immediate_operand" "i")]
2859          VSHLLTQ_N))
2860   ]
2861   "TARGET_HAVE_MVE"
2862   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2863   [(set_attr "type" "mve_move")
2864 ])
2865
2866 ;;
2867 ;; [vsubq_f])
2868 ;;
2869 (define_insn "mve_vsubq_f<mode>"
2870   [
2871    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2872         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2873                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2874   ]
2875   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2876   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2877   [(set_attr "type" "mve_move")
2878 ])
2879
2880 ;;
2881 ;; [vmulltq_poly_p])
2882 ;;
2883 (define_insn "mve_vmulltq_poly_p<mode>"
2884   [
2885    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2886         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2887                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2888          VMULLTQ_POLY_P))
2889   ]
2890   "TARGET_HAVE_MVE"
2891   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2892   [(set_attr "type" "mve_move")
2893 ])
2894
2895 ;;
2896 ;; [vmullbq_poly_p])
2897 ;;
2898 (define_insn "mve_vmullbq_poly_p<mode>"
2899   [
2900    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2901         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2902                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2903          VMULLBQ_POLY_P))
2904   ]
2905   "TARGET_HAVE_MVE"
2906   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2907   [(set_attr "type" "mve_move")
2908 ])
2909
2910 ;;
2911 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2912 ;;
2913 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2914   [
2915    (set (match_operand:DI 0 "s_register_operand" "=r")
2916         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2917                     (match_operand:V4SI 2 "s_register_operand" "w")]
2918          VRMLALDAVHQ))
2919   ]
2920   "TARGET_HAVE_MVE"
2921   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2922   [(set_attr "type" "mve_move")
2923 ])
2924
2925 ;;
2926 ;; [vbicq_m_n_s, vbicq_m_n_u])
2927 ;;
2928 (define_insn "mve_vbicq_m_n_<supf><mode>"
2929   [
2930    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2931         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2932                        (match_operand:SI 2 "immediate_operand" "i")
2933                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2934          VBICQ_M_N))
2935   ]
2936   "TARGET_HAVE_MVE"
2937   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2938   [(set_attr "type" "mve_move")
2939    (set_attr "length""8")])
2940 ;;
2941 ;; [vcmpeqq_m_f])
2942 ;;
2943 (define_insn "mve_vcmpeqq_m_f<mode>"
2944   [
2945    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2946         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2947                     (match_operand:MVE_0 2 "s_register_operand" "w")
2948                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2949          VCMPEQQ_M_F))
2950   ]
2951   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2952   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2953   [(set_attr "type" "mve_move")
2954    (set_attr "length""8")])
2955 ;;
2956 ;; [vcvtaq_m_u, vcvtaq_m_s])
2957 ;;
2958 (define_insn "mve_vcvtaq_m_<supf><mode>"
2959   [
2960    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2961         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2962                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2963                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2964          VCVTAQ_M))
2965   ]
2966   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2967   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2968   [(set_attr "type" "mve_move")
2969    (set_attr "length""8")])
2970 ;;
2971 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2972 ;;
2973 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2974   [
2975    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2976         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2977                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2978                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2979          VCVTQ_M_TO_F))
2980   ]
2981   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2982   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2983   [(set_attr "type" "mve_move")
2984    (set_attr "length""8")])
2985 ;;
2986 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2987 ;;
2988 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2989   [
2990    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2991         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2992                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2993                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2994          VQRSHRNBQ_N))
2995   ]
2996   "TARGET_HAVE_MVE"
2997   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2998   [(set_attr "type" "mve_move")
2999 ])
3000 ;;
3001 ;; [vqrshrunbq_n_s])
3002 ;;
3003 (define_insn "mve_vqrshrunbq_n_s<mode>"
3004   [
3005    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3006         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3007                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3008                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3009          VQRSHRUNBQ_N_S))
3010   ]
3011   "TARGET_HAVE_MVE"
3012   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3013   [(set_attr "type" "mve_move")
3014 ])
3015 ;;
3016 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3017 ;;
3018 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3019   [
3020    (set (match_operand:DI 0 "s_register_operand" "=r")
3021         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3022                     (match_operand:V4SI 2 "s_register_operand" "w")
3023                     (match_operand:V4SI 3 "s_register_operand" "w")]
3024          VRMLALDAVHAQ))
3025   ]
3026   "TARGET_HAVE_MVE"
3027   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3028   [(set_attr "type" "mve_move")
3029 ])
3030
3031 ;;
3032 ;; [vabavq_s, vabavq_u])
3033 ;;
3034 (define_insn "mve_vabavq_<supf><mode>"
3035   [
3036    (set (match_operand:SI 0 "s_register_operand" "=r")
3037         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3038                     (match_operand:MVE_2 2 "s_register_operand" "w")
3039                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3040          VABAVQ))
3041   ]
3042   "TARGET_HAVE_MVE"
3043   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3044   [(set_attr "type" "mve_move")
3045 ])
3046
3047 ;;
3048 ;; [vshlcq_u vshlcq_s]
3049 ;;
3050 (define_expand "mve_vshlcq_vec_<supf><mode>"
3051  [(match_operand:MVE_2 0 "s_register_operand")
3052   (match_operand:MVE_2 1 "s_register_operand")
3053   (match_operand:SI 2 "s_register_operand")
3054   (match_operand:SI 3 "mve_imm_32")
3055   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3056  "TARGET_HAVE_MVE"
3057 {
3058   rtx ignore_wb = gen_reg_rtx (SImode);
3059   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3060                                       operands[2], operands[3]));
3061   DONE;
3062 })
3063
3064 (define_expand "mve_vshlcq_carry_<supf><mode>"
3065  [(match_operand:SI 0 "s_register_operand")
3066   (match_operand:MVE_2 1 "s_register_operand")
3067   (match_operand:SI 2 "s_register_operand")
3068   (match_operand:SI 3 "mve_imm_32")
3069   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3070  "TARGET_HAVE_MVE"
3071 {
3072   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3073   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3074                                       operands[2], operands[3]));
3075   DONE;
3076 })
3077
3078 (define_insn "mve_vshlcq_<supf><mode>"
3079  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3080        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3081                       (match_operand:SI 3 "s_register_operand" "1")
3082                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3083         VSHLCQ))
3084   (set (match_operand:SI  1 "s_register_operand" "=r")
3085        (unspec:SI [(match_dup 2)
3086                    (match_dup 3)
3087                    (match_dup 4)]
3088         VSHLCQ))]
3089  "TARGET_HAVE_MVE"
3090  "vshlc %q0, %1, %4")
3091
3092 ;;
3093 ;; [vabsq_m_s])
3094 ;;
3095 (define_insn "mve_vabsq_m_s<mode>"
3096   [
3097    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3098         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3099                        (match_operand:MVE_2 2 "s_register_operand" "w")
3100                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3101          VABSQ_M_S))
3102   ]
3103   "TARGET_HAVE_MVE"
3104   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3105   [(set_attr "type" "mve_move")
3106    (set_attr "length""8")])
3107
3108 ;;
3109 ;; [vaddvaq_p_u, vaddvaq_p_s])
3110 ;;
3111 (define_insn "mve_vaddvaq_p_<supf><mode>"
3112   [
3113    (set (match_operand:SI 0 "s_register_operand" "=Te")
3114         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3115                        (match_operand:MVE_2 2 "s_register_operand" "w")
3116                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3117          VADDVAQ_P))
3118   ]
3119   "TARGET_HAVE_MVE"
3120   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3121   [(set_attr "type" "mve_move")
3122    (set_attr "length""8")])
3123
3124 ;;
3125 ;; [vclsq_m_s])
3126 ;;
3127 (define_insn "mve_vclsq_m_s<mode>"
3128   [
3129    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3130         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3131                        (match_operand:MVE_2 2 "s_register_operand" "w")
3132                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3133          VCLSQ_M_S))
3134   ]
3135   "TARGET_HAVE_MVE"
3136   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3137   [(set_attr "type" "mve_move")
3138    (set_attr "length""8")])
3139
3140 ;;
3141 ;; [vclzq_m_s, vclzq_m_u])
3142 ;;
3143 (define_insn "mve_vclzq_m_<supf><mode>"
3144   [
3145    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3146         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3147                        (match_operand:MVE_2 2 "s_register_operand" "w")
3148                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3149          VCLZQ_M))
3150   ]
3151   "TARGET_HAVE_MVE"
3152   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3153   [(set_attr "type" "mve_move")
3154    (set_attr "length""8")])
3155
3156 ;;
3157 ;; [vcmpcsq_m_n_u])
3158 ;;
3159 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3160   [
3161    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3162         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3163                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3164                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3165          VCMPCSQ_M_N_U))
3166   ]
3167   "TARGET_HAVE_MVE"
3168   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3169   [(set_attr "type" "mve_move")
3170    (set_attr "length""8")])
3171
3172 ;;
3173 ;; [vcmpcsq_m_u])
3174 ;;
3175 (define_insn "mve_vcmpcsq_m_u<mode>"
3176   [
3177    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3178         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3179                        (match_operand:MVE_2 2 "s_register_operand" "w")
3180                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3181          VCMPCSQ_M_U))
3182   ]
3183   "TARGET_HAVE_MVE"
3184   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3185   [(set_attr "type" "mve_move")
3186    (set_attr "length""8")])
3187
3188 ;;
3189 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3190 ;;
3191 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3192   [
3193    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3194         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3195                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3196                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3197          VCMPEQQ_M_N))
3198   ]
3199   "TARGET_HAVE_MVE"
3200   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3201   [(set_attr "type" "mve_move")
3202    (set_attr "length""8")])
3203
3204 ;;
3205 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3206 ;;
3207 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3208   [
3209    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3210         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3211                        (match_operand:MVE_2 2 "s_register_operand" "w")
3212                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3213          VCMPEQQ_M))
3214   ]
3215   "TARGET_HAVE_MVE"
3216   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3217   [(set_attr "type" "mve_move")
3218    (set_attr "length""8")])
3219
3220 ;;
3221 ;; [vcmpgeq_m_n_s])
3222 ;;
3223 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3224   [
3225    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3226         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3227                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3228                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3229          VCMPGEQ_M_N_S))
3230   ]
3231   "TARGET_HAVE_MVE"
3232   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3233   [(set_attr "type" "mve_move")
3234    (set_attr "length""8")])
3235
3236 ;;
3237 ;; [vcmpgeq_m_s])
3238 ;;
3239 (define_insn "mve_vcmpgeq_m_s<mode>"
3240   [
3241    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3242         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3243                        (match_operand:MVE_2 2 "s_register_operand" "w")
3244                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3245          VCMPGEQ_M_S))
3246   ]
3247   "TARGET_HAVE_MVE"
3248   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3249   [(set_attr "type" "mve_move")
3250    (set_attr "length""8")])
3251
3252 ;;
3253 ;; [vcmpgtq_m_n_s])
3254 ;;
3255 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3256   [
3257    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3258         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3259                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3260                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3261          VCMPGTQ_M_N_S))
3262   ]
3263   "TARGET_HAVE_MVE"
3264   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3265   [(set_attr "type" "mve_move")
3266    (set_attr "length""8")])
3267
3268 ;;
3269 ;; [vcmpgtq_m_s])
3270 ;;
3271 (define_insn "mve_vcmpgtq_m_s<mode>"
3272   [
3273    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3274         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3275                        (match_operand:MVE_2 2 "s_register_operand" "w")
3276                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3277          VCMPGTQ_M_S))
3278   ]
3279   "TARGET_HAVE_MVE"
3280   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3281   [(set_attr "type" "mve_move")
3282    (set_attr "length""8")])
3283
3284 ;;
3285 ;; [vcmphiq_m_n_u])
3286 ;;
3287 (define_insn "mve_vcmphiq_m_n_u<mode>"
3288   [
3289    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3290         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3291                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3292                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3293          VCMPHIQ_M_N_U))
3294   ]
3295   "TARGET_HAVE_MVE"
3296   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3297   [(set_attr "type" "mve_move")
3298    (set_attr "length""8")])
3299
3300 ;;
3301 ;; [vcmphiq_m_u])
3302 ;;
3303 (define_insn "mve_vcmphiq_m_u<mode>"
3304   [
3305    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3306         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3307                        (match_operand:MVE_2 2 "s_register_operand" "w")
3308                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3309          VCMPHIQ_M_U))
3310   ]
3311   "TARGET_HAVE_MVE"
3312   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3313   [(set_attr "type" "mve_move")
3314    (set_attr "length""8")])
3315
3316 ;;
3317 ;; [vcmpleq_m_n_s])
3318 ;;
3319 (define_insn "mve_vcmpleq_m_n_s<mode>"
3320   [
3321    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3322         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3323                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3324                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3325          VCMPLEQ_M_N_S))
3326   ]
3327   "TARGET_HAVE_MVE"
3328   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3329   [(set_attr "type" "mve_move")
3330    (set_attr "length""8")])
3331
3332 ;;
3333 ;; [vcmpleq_m_s])
3334 ;;
3335 (define_insn "mve_vcmpleq_m_s<mode>"
3336   [
3337    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3338         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3339                        (match_operand:MVE_2 2 "s_register_operand" "w")
3340                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3341          VCMPLEQ_M_S))
3342   ]
3343   "TARGET_HAVE_MVE"
3344   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3345   [(set_attr "type" "mve_move")
3346    (set_attr "length""8")])
3347
3348 ;;
3349 ;; [vcmpltq_m_n_s])
3350 ;;
3351 (define_insn "mve_vcmpltq_m_n_s<mode>"
3352   [
3353    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3354         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3355                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3356                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3357          VCMPLTQ_M_N_S))
3358   ]
3359   "TARGET_HAVE_MVE"
3360   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3361   [(set_attr "type" "mve_move")
3362    (set_attr "length""8")])
3363
3364 ;;
3365 ;; [vcmpltq_m_s])
3366 ;;
3367 (define_insn "mve_vcmpltq_m_s<mode>"
3368   [
3369    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3370         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3371                        (match_operand:MVE_2 2 "s_register_operand" "w")
3372                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3373          VCMPLTQ_M_S))
3374   ]
3375   "TARGET_HAVE_MVE"
3376   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3377   [(set_attr "type" "mve_move")
3378    (set_attr "length""8")])
3379
3380 ;;
3381 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3382 ;;
3383 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3384   [
3385    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3386         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3387                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3388                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3389          VCMPNEQ_M_N))
3390   ]
3391   "TARGET_HAVE_MVE"
3392   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3393   [(set_attr "type" "mve_move")
3394    (set_attr "length""8")])
3395
3396 ;;
3397 ;; [vcmpneq_m_s, vcmpneq_m_u])
3398 ;;
3399 (define_insn "mve_vcmpneq_m_<supf><mode>"
3400   [
3401    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3402         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3403                        (match_operand:MVE_2 2 "s_register_operand" "w")
3404                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3405          VCMPNEQ_M))
3406   ]
3407   "TARGET_HAVE_MVE"
3408   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3409   [(set_attr "type" "mve_move")
3410    (set_attr "length""8")])
3411
3412 ;;
3413 ;; [vdupq_m_n_s, vdupq_m_n_u])
3414 ;;
3415 (define_insn "mve_vdupq_m_n_<supf><mode>"
3416   [
3417    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3418         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3419                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3420                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3421          VDUPQ_M_N))
3422   ]
3423   "TARGET_HAVE_MVE"
3424   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3425   [(set_attr "type" "mve_move")
3426    (set_attr "length""8")])
3427
3428 ;;
3429 ;; [vmaxaq_m_s])
3430 ;;
3431 (define_insn "mve_vmaxaq_m_s<mode>"
3432   [
3433    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3434         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3435                        (match_operand:MVE_2 2 "s_register_operand" "w")
3436                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3437          VMAXAQ_M_S))
3438   ]
3439   "TARGET_HAVE_MVE"
3440   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3441   [(set_attr "type" "mve_move")
3442    (set_attr "length""8")])
3443
3444 ;;
3445 ;; [vmaxavq_p_s])
3446 ;;
3447 (define_insn "mve_vmaxavq_p_s<mode>"
3448   [
3449    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3450         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3451                        (match_operand:MVE_2 2 "s_register_operand" "w")
3452                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3453          VMAXAVQ_P_S))
3454   ]
3455   "TARGET_HAVE_MVE"
3456   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3457   [(set_attr "type" "mve_move")
3458    (set_attr "length""8")])
3459
3460 ;;
3461 ;; [vmaxvq_p_u, vmaxvq_p_s])
3462 ;;
3463 (define_insn "mve_vmaxvq_p_<supf><mode>"
3464   [
3465    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3466         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3467                        (match_operand:MVE_2 2 "s_register_operand" "w")
3468                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3469          VMAXVQ_P))
3470   ]
3471   "TARGET_HAVE_MVE"
3472   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3473   [(set_attr "type" "mve_move")
3474    (set_attr "length""8")])
3475
3476 ;;
3477 ;; [vminaq_m_s])
3478 ;;
3479 (define_insn "mve_vminaq_m_s<mode>"
3480   [
3481    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3482         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3483                        (match_operand:MVE_2 2 "s_register_operand" "w")
3484                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3485          VMINAQ_M_S))
3486   ]
3487   "TARGET_HAVE_MVE"
3488   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3489   [(set_attr "type" "mve_move")
3490    (set_attr "length""8")])
3491
3492 ;;
3493 ;; [vminavq_p_s])
3494 ;;
3495 (define_insn "mve_vminavq_p_s<mode>"
3496   [
3497    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3498         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3499                        (match_operand:MVE_2 2 "s_register_operand" "w")
3500                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3501          VMINAVQ_P_S))
3502   ]
3503   "TARGET_HAVE_MVE"
3504   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3505   [(set_attr "type" "mve_move")
3506    (set_attr "length""8")])
3507
3508 ;;
3509 ;; [vminvq_p_s, vminvq_p_u])
3510 ;;
3511 (define_insn "mve_vminvq_p_<supf><mode>"
3512   [
3513    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3514         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3515                        (match_operand:MVE_2 2 "s_register_operand" "w")
3516                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3517          VMINVQ_P))
3518   ]
3519   "TARGET_HAVE_MVE"
3520   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3521   [(set_attr "type" "mve_move")
3522    (set_attr "length""8")])
3523
3524 ;;
3525 ;; [vmladavaq_u, vmladavaq_s])
3526 ;;
3527 (define_insn "mve_vmladavaq_<supf><mode>"
3528   [
3529    (set (match_operand:SI 0 "s_register_operand" "=Te")
3530         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3531                        (match_operand:MVE_2 2 "s_register_operand" "w")
3532                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3533          VMLADAVAQ))
3534   ]
3535   "TARGET_HAVE_MVE"
3536   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3537   [(set_attr "type" "mve_move")
3538 ])
3539
3540 ;;
3541 ;; [vmladavq_p_u, vmladavq_p_s])
3542 ;;
3543 (define_insn "mve_vmladavq_p_<supf><mode>"
3544   [
3545    (set (match_operand:SI 0 "s_register_operand" "=Te")
3546         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3547                        (match_operand:MVE_2 2 "s_register_operand" "w")
3548                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3549          VMLADAVQ_P))
3550   ]
3551   "TARGET_HAVE_MVE"
3552   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3553   [(set_attr "type" "mve_move")
3554    (set_attr "length""8")])
3555
3556 ;;
3557 ;; [vmladavxq_p_s])
3558 ;;
3559 (define_insn "mve_vmladavxq_p_s<mode>"
3560   [
3561    (set (match_operand:SI 0 "s_register_operand" "=Te")
3562         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3563                        (match_operand:MVE_2 2 "s_register_operand" "w")
3564                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3565          VMLADAVXQ_P_S))
3566   ]
3567   "TARGET_HAVE_MVE"
3568   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3569   [(set_attr "type" "mve_move")
3570    (set_attr "length""8")])
3571
3572 ;;
3573 ;; [vmlaq_n_u, vmlaq_n_s])
3574 ;;
3575 (define_insn "mve_vmlaq_n_<supf><mode>"
3576   [
3577    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3578         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3579                        (match_operand:MVE_2 2 "s_register_operand" "w")
3580                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3581          VMLAQ_N))
3582   ]
3583   "TARGET_HAVE_MVE"
3584   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3585   [(set_attr "type" "mve_move")
3586 ])
3587
3588 ;;
3589 ;; [vmlasq_n_u, vmlasq_n_s])
3590 ;;
3591 (define_insn "mve_vmlasq_n_<supf><mode>"
3592   [
3593    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3594         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3595                        (match_operand:MVE_2 2 "s_register_operand" "w")
3596                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3597          VMLASQ_N))
3598   ]
3599   "TARGET_HAVE_MVE"
3600   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3601   [(set_attr "type" "mve_move")
3602 ])
3603
3604 ;;
3605 ;; [vmlsdavq_p_s])
3606 ;;
3607 (define_insn "mve_vmlsdavq_p_s<mode>"
3608   [
3609    (set (match_operand:SI 0 "s_register_operand" "=Te")
3610         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3611                        (match_operand:MVE_2 2 "s_register_operand" "w")
3612                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3613          VMLSDAVQ_P_S))
3614   ]
3615   "TARGET_HAVE_MVE"
3616   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3617   [(set_attr "type" "mve_move")
3618    (set_attr "length""8")])
3619
3620 ;;
3621 ;; [vmlsdavxq_p_s])
3622 ;;
3623 (define_insn "mve_vmlsdavxq_p_s<mode>"
3624   [
3625    (set (match_operand:SI 0 "s_register_operand" "=Te")
3626         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3627                        (match_operand:MVE_2 2 "s_register_operand" "w")
3628                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3629          VMLSDAVXQ_P_S))
3630   ]
3631   "TARGET_HAVE_MVE"
3632   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3633   [(set_attr "type" "mve_move")
3634    (set_attr "length""8")])
3635
3636 ;;
3637 ;; [vmvnq_m_s, vmvnq_m_u])
3638 ;;
3639 (define_insn "mve_vmvnq_m_<supf><mode>"
3640   [
3641    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3642         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3643                        (match_operand:MVE_2 2 "s_register_operand" "w")
3644                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3645          VMVNQ_M))
3646   ]
3647   "TARGET_HAVE_MVE"
3648   "vpst\;vmvnt %q0, %q2"
3649   [(set_attr "type" "mve_move")
3650    (set_attr "length""8")])
3651
3652 ;;
3653 ;; [vnegq_m_s])
3654 ;;
3655 (define_insn "mve_vnegq_m_s<mode>"
3656   [
3657    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3658         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3659                        (match_operand:MVE_2 2 "s_register_operand" "w")
3660                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3661          VNEGQ_M_S))
3662   ]
3663   "TARGET_HAVE_MVE"
3664   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3665   [(set_attr "type" "mve_move")
3666    (set_attr "length""8")])
3667
3668 ;;
3669 ;; [vpselq_u, vpselq_s])
3670 ;;
3671 (define_insn "mve_vpselq_<supf><mode>"
3672   [
3673    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3674         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3675                        (match_operand:MVE_1 2 "s_register_operand" "w")
3676                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3677          VPSELQ))
3678   ]
3679   "TARGET_HAVE_MVE"
3680   "vpsel %q0, %q1, %q2"
3681   [(set_attr "type" "mve_move")
3682 ])
3683
3684 ;;
3685 ;; [vqabsq_m_s])
3686 ;;
3687 (define_insn "mve_vqabsq_m_s<mode>"
3688   [
3689    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3690         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3691                        (match_operand:MVE_2 2 "s_register_operand" "w")
3692                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3693          VQABSQ_M_S))
3694   ]
3695   "TARGET_HAVE_MVE"
3696   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3697   [(set_attr "type" "mve_move")
3698    (set_attr "length""8")])
3699
3700 ;;
3701 ;; [vqdmlahq_n_s])
3702 ;;
3703 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3704   [
3705    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3706         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3707                        (match_operand:MVE_2 2 "s_register_operand" "w")
3708                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3709          VQDMLAHQ_N))
3710   ]
3711   "TARGET_HAVE_MVE"
3712   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3713   [(set_attr "type" "mve_move")
3714 ])
3715
3716 ;;
3717 ;; [vqdmlashq_n_s])
3718 ;;
3719 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3720   [
3721    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3722         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3723                        (match_operand:MVE_2 2 "s_register_operand" "w")
3724                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3725          VQDMLASHQ_N))
3726   ]
3727   "TARGET_HAVE_MVE"
3728   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3729   [(set_attr "type" "mve_move")
3730 ])
3731
3732 ;;
3733 ;; [vqnegq_m_s])
3734 ;;
3735 (define_insn "mve_vqnegq_m_s<mode>"
3736   [
3737    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3738         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3739                        (match_operand:MVE_2 2 "s_register_operand" "w")
3740                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3741          VQNEGQ_M_S))
3742   ]
3743   "TARGET_HAVE_MVE"
3744   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3745   [(set_attr "type" "mve_move")
3746    (set_attr "length""8")])
3747
3748 ;;
3749 ;; [vqrdmladhq_s])
3750 ;;
3751 (define_insn "mve_vqrdmladhq_s<mode>"
3752   [
3753    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3754         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3755                        (match_operand:MVE_2 2 "s_register_operand" "w")
3756                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3757          VQRDMLADHQ_S))
3758   ]
3759   "TARGET_HAVE_MVE"
3760   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3761   [(set_attr "type" "mve_move")
3762 ])
3763
3764 ;;
3765 ;; [vqrdmladhxq_s])
3766 ;;
3767 (define_insn "mve_vqrdmladhxq_s<mode>"
3768   [
3769    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3770         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3771                        (match_operand:MVE_2 2 "s_register_operand" "w")
3772                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3773          VQRDMLADHXQ_S))
3774   ]
3775   "TARGET_HAVE_MVE"
3776   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3777   [(set_attr "type" "mve_move")
3778 ])
3779
3780 ;;
3781 ;; [vqrdmlahq_n_s])
3782 ;;
3783 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3784   [
3785    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3786         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3787                        (match_operand:MVE_2 2 "s_register_operand" "w")
3788                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3789          VQRDMLAHQ_N))
3790   ]
3791   "TARGET_HAVE_MVE"
3792   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3793   [(set_attr "type" "mve_move")
3794 ])
3795
3796 ;;
3797 ;; [vqrdmlashq_n_s])
3798 ;;
3799 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3800   [
3801    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3802         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3803                        (match_operand:MVE_2 2 "s_register_operand" "w")
3804                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3805          VQRDMLASHQ_N))
3806   ]
3807   "TARGET_HAVE_MVE"
3808   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3809   [(set_attr "type" "mve_move")
3810 ])
3811
3812 ;;
3813 ;; [vqrdmlsdhq_s])
3814 ;;
3815 (define_insn "mve_vqrdmlsdhq_s<mode>"
3816   [
3817    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3818         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3819                        (match_operand:MVE_2 2 "s_register_operand" "w")
3820                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3821          VQRDMLSDHQ_S))
3822   ]
3823   "TARGET_HAVE_MVE"
3824   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3825   [(set_attr "type" "mve_move")
3826 ])
3827
3828 ;;
3829 ;; [vqrdmlsdhxq_s])
3830 ;;
3831 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3832   [
3833    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3834         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3835                        (match_operand:MVE_2 2 "s_register_operand" "w")
3836                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3837          VQRDMLSDHXQ_S))
3838   ]
3839   "TARGET_HAVE_MVE"
3840   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3841   [(set_attr "type" "mve_move")
3842 ])
3843
3844 ;;
3845 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3846 ;;
3847 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3848   [
3849    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3850         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3851                        (match_operand:SI 2 "s_register_operand" "r")
3852                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3853          VQRSHLQ_M_N))
3854   ]
3855   "TARGET_HAVE_MVE"
3856   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3857   [(set_attr "type" "mve_move")
3858    (set_attr "length""8")])
3859
3860 ;;
3861 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3862 ;;
3863 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3864   [
3865    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3866         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3867                        (match_operand:SI 2 "s_register_operand" "r")
3868                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3869          VQSHLQ_M_R))
3870   ]
3871   "TARGET_HAVE_MVE"
3872   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3873   [(set_attr "type" "mve_move")
3874    (set_attr "length""8")])
3875
3876 ;;
3877 ;; [vrev64q_m_u, vrev64q_m_s])
3878 ;;
3879 (define_insn "mve_vrev64q_m_<supf><mode>"
3880   [
3881    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3882         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3883                        (match_operand:MVE_2 2 "s_register_operand" "w")
3884                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3885          VREV64Q_M))
3886   ]
3887   "TARGET_HAVE_MVE"
3888   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3889   [(set_attr "type" "mve_move")
3890    (set_attr "length""8")])
3891
3892 ;;
3893 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3894 ;;
3895 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3896   [
3897    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3898         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3899                        (match_operand:SI 2 "s_register_operand" "r")
3900                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3901          VRSHLQ_M_N))
3902   ]
3903   "TARGET_HAVE_MVE"
3904   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3905   [(set_attr "type" "mve_move")
3906    (set_attr "length""8")])
3907
3908 ;;
3909 ;; [vshlq_m_r_u, vshlq_m_r_s])
3910 ;;
3911 (define_insn "mve_vshlq_m_r_<supf><mode>"
3912   [
3913    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3914         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3915                        (match_operand:SI 2 "s_register_operand" "r")
3916                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3917          VSHLQ_M_R))
3918   ]
3919   "TARGET_HAVE_MVE"
3920   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3921   [(set_attr "type" "mve_move")
3922    (set_attr "length""8")])
3923
3924 ;;
3925 ;; [vsliq_n_u, vsliq_n_s])
3926 ;;
3927 (define_insn "mve_vsliq_n_<supf><mode>"
3928   [
3929    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3930         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3931                        (match_operand:MVE_2 2 "s_register_operand" "w")
3932                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3933          VSLIQ_N))
3934   ]
3935   "TARGET_HAVE_MVE"
3936   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3937   [(set_attr "type" "mve_move")
3938 ])
3939
3940 ;;
3941 ;; [vsriq_n_u, vsriq_n_s])
3942 ;;
3943 (define_insn "mve_vsriq_n_<supf><mode>"
3944   [
3945    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3946         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3947                        (match_operand:MVE_2 2 "s_register_operand" "w")
3948                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3949          VSRIQ_N))
3950   ]
3951   "TARGET_HAVE_MVE"
3952   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3953   [(set_attr "type" "mve_move")
3954 ])
3955
3956 ;;
3957 ;; [vqdmlsdhxq_s])
3958 ;;
3959 (define_insn "mve_vqdmlsdhxq_s<mode>"
3960   [
3961    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3962         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3963                        (match_operand:MVE_2 2 "s_register_operand" "w")
3964                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3965          VQDMLSDHXQ_S))
3966   ]
3967   "TARGET_HAVE_MVE"
3968   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3969   [(set_attr "type" "mve_move")
3970 ])
3971
3972 ;;
3973 ;; [vqdmlsdhq_s])
3974 ;;
3975 (define_insn "mve_vqdmlsdhq_s<mode>"
3976   [
3977    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3978         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3979                        (match_operand:MVE_2 2 "s_register_operand" "w")
3980                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3981          VQDMLSDHQ_S))
3982   ]
3983   "TARGET_HAVE_MVE"
3984   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3985   [(set_attr "type" "mve_move")
3986 ])
3987
3988 ;;
3989 ;; [vqdmladhxq_s])
3990 ;;
3991 (define_insn "mve_vqdmladhxq_s<mode>"
3992   [
3993    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3994         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3995                        (match_operand:MVE_2 2 "s_register_operand" "w")
3996                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3997          VQDMLADHXQ_S))
3998   ]
3999   "TARGET_HAVE_MVE"
4000   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4001   [(set_attr "type" "mve_move")
4002 ])
4003
4004 ;;
4005 ;; [vqdmladhq_s])
4006 ;;
4007 (define_insn "mve_vqdmladhq_s<mode>"
4008   [
4009    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4010         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4011                        (match_operand:MVE_2 2 "s_register_operand" "w")
4012                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4013          VQDMLADHQ_S))
4014   ]
4015   "TARGET_HAVE_MVE"
4016   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4017   [(set_attr "type" "mve_move")
4018 ])
4019
4020 ;;
4021 ;; [vmlsdavaxq_s])
4022 ;;
4023 (define_insn "mve_vmlsdavaxq_s<mode>"
4024   [
4025    (set (match_operand:SI 0 "s_register_operand" "=Te")
4026         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4027                     (match_operand:MVE_2 2 "s_register_operand" "w")
4028                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4029          VMLSDAVAXQ_S))
4030   ]
4031   "TARGET_HAVE_MVE"
4032   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4033   [(set_attr "type" "mve_move")
4034 ])
4035
4036 ;;
4037 ;; [vmlsdavaq_s])
4038 ;;
4039 (define_insn "mve_vmlsdavaq_s<mode>"
4040   [
4041    (set (match_operand:SI 0 "s_register_operand" "=Te")
4042         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4043                     (match_operand:MVE_2 2 "s_register_operand" "w")
4044                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4045          VMLSDAVAQ_S))
4046   ]
4047   "TARGET_HAVE_MVE"
4048   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4049   [(set_attr "type" "mve_move")
4050 ])
4051
4052 ;;
4053 ;; [vmladavaxq_s])
4054 ;;
4055 (define_insn "mve_vmladavaxq_s<mode>"
4056   [
4057    (set (match_operand:SI 0 "s_register_operand" "=Te")
4058         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4059                     (match_operand:MVE_2 2 "s_register_operand" "w")
4060                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4061          VMLADAVAXQ_S))
4062   ]
4063   "TARGET_HAVE_MVE"
4064   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4065   [(set_attr "type" "mve_move")
4066 ])
4067 ;;
4068 ;; [vabsq_m_f])
4069 ;;
4070 (define_insn "mve_vabsq_m_f<mode>"
4071   [
4072    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4073         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4074                        (match_operand:MVE_0 2 "s_register_operand" "w")
4075                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4076          VABSQ_M_F))
4077   ]
4078   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4079   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4080   [(set_attr "type" "mve_move")
4081    (set_attr "length""8")])
4082
4083 ;;
4084 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4085 ;;
4086 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4087   [
4088    (set (match_operand:DI 0 "s_register_operand" "=r")
4089         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4090                        (match_operand:V4SI 2 "s_register_operand" "w")
4091                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4092          VADDLVAQ_P))
4093   ]
4094   "TARGET_HAVE_MVE"
4095   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4096   [(set_attr "type" "mve_move")
4097    (set_attr "length""8")])
4098 ;;
4099 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
4100 ;;
4101 (define_insn "mve_vcmlaq<mve_rot><mode>"
4102   [
4103    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
4104         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
4105                     (unspec:MVE_0
4106                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
4107                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
4108                      VCMLA)))
4109   ]
4110   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4111   "@
4112    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
4113    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
4114   [(set_attr "type" "mve_move")
4115 ])
4116
4117 ;;
4118 ;; [vcmpeqq_m_n_f])
4119 ;;
4120 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4121   [
4122    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4123         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4124                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4125                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4126          VCMPEQQ_M_N_F))
4127   ]
4128   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4129   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4130   [(set_attr "type" "mve_move")
4131    (set_attr "length""8")])
4132
4133 ;;
4134 ;; [vcmpgeq_m_f])
4135 ;;
4136 (define_insn "mve_vcmpgeq_m_f<mode>"
4137   [
4138    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4139         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4140                        (match_operand:MVE_0 2 "s_register_operand" "w")
4141                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4142          VCMPGEQ_M_F))
4143   ]
4144   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4145   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4146   [(set_attr "type" "mve_move")
4147    (set_attr "length""8")])
4148
4149 ;;
4150 ;; [vcmpgeq_m_n_f])
4151 ;;
4152 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4153   [
4154    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4155         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4156                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4157                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4158          VCMPGEQ_M_N_F))
4159   ]
4160   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4161   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4162   [(set_attr "type" "mve_move")
4163    (set_attr "length""8")])
4164
4165 ;;
4166 ;; [vcmpgtq_m_f])
4167 ;;
4168 (define_insn "mve_vcmpgtq_m_f<mode>"
4169   [
4170    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4171         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4172                        (match_operand:MVE_0 2 "s_register_operand" "w")
4173                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4174          VCMPGTQ_M_F))
4175   ]
4176   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4177   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4178   [(set_attr "type" "mve_move")
4179    (set_attr "length""8")])
4180
4181 ;;
4182 ;; [vcmpgtq_m_n_f])
4183 ;;
4184 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4185   [
4186    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4187         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4188                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4189                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4190          VCMPGTQ_M_N_F))
4191   ]
4192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4193   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4194   [(set_attr "type" "mve_move")
4195    (set_attr "length""8")])
4196
4197 ;;
4198 ;; [vcmpleq_m_f])
4199 ;;
4200 (define_insn "mve_vcmpleq_m_f<mode>"
4201   [
4202    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4203         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4204                        (match_operand:MVE_0 2 "s_register_operand" "w")
4205                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4206          VCMPLEQ_M_F))
4207   ]
4208   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4209   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4210   [(set_attr "type" "mve_move")
4211    (set_attr "length""8")])
4212
4213 ;;
4214 ;; [vcmpleq_m_n_f])
4215 ;;
4216 (define_insn "mve_vcmpleq_m_n_f<mode>"
4217   [
4218    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4219         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4220                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4221                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4222          VCMPLEQ_M_N_F))
4223   ]
4224   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4225   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4226   [(set_attr "type" "mve_move")
4227    (set_attr "length""8")])
4228
4229 ;;
4230 ;; [vcmpltq_m_f])
4231 ;;
4232 (define_insn "mve_vcmpltq_m_f<mode>"
4233   [
4234    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4235         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4236                        (match_operand:MVE_0 2 "s_register_operand" "w")
4237                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4238          VCMPLTQ_M_F))
4239   ]
4240   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4241   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4242   [(set_attr "type" "mve_move")
4243    (set_attr "length""8")])
4244
4245 ;;
4246 ;; [vcmpltq_m_n_f])
4247 ;;
4248 (define_insn "mve_vcmpltq_m_n_f<mode>"
4249   [
4250    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4251         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4252                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4253                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4254          VCMPLTQ_M_N_F))
4255   ]
4256   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4257   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4258   [(set_attr "type" "mve_move")
4259    (set_attr "length""8")])
4260
4261 ;;
4262 ;; [vcmpneq_m_f])
4263 ;;
4264 (define_insn "mve_vcmpneq_m_f<mode>"
4265   [
4266    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4267         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4268                        (match_operand:MVE_0 2 "s_register_operand" "w")
4269                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4270          VCMPNEQ_M_F))
4271   ]
4272   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4273   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4274   [(set_attr "type" "mve_move")
4275    (set_attr "length""8")])
4276
4277 ;;
4278 ;; [vcmpneq_m_n_f])
4279 ;;
4280 (define_insn "mve_vcmpneq_m_n_f<mode>"
4281   [
4282    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4283         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4284                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4285                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4286          VCMPNEQ_M_N_F))
4287   ]
4288   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4289   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4290   [(set_attr "type" "mve_move")
4291    (set_attr "length""8")])
4292
4293 ;;
4294 ;; [vcvtbq_m_f16_f32])
4295 ;;
4296 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4297   [
4298    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4299         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4300                        (match_operand:V4SF 2 "s_register_operand" "w")
4301                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4302          VCVTBQ_M_F16_F32))
4303   ]
4304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4305   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4306   [(set_attr "type" "mve_move")
4307    (set_attr "length""8")])
4308
4309 ;;
4310 ;; [vcvtbq_m_f32_f16])
4311 ;;
4312 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4313   [
4314    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4315         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4316                        (match_operand:V8HF 2 "s_register_operand" "w")
4317                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4318          VCVTBQ_M_F32_F16))
4319   ]
4320   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4321   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4322   [(set_attr "type" "mve_move")
4323    (set_attr "length""8")])
4324
4325 ;;
4326 ;; [vcvttq_m_f16_f32])
4327 ;;
4328 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4329   [
4330    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4331         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4332                        (match_operand:V4SF 2 "s_register_operand" "w")
4333                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4334          VCVTTQ_M_F16_F32))
4335   ]
4336   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4337   "vpst\;vcvttt.f16.f32 %q0, %q2"
4338   [(set_attr "type" "mve_move")
4339    (set_attr "length""8")])
4340
4341 ;;
4342 ;; [vcvttq_m_f32_f16])
4343 ;;
4344 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4345   [
4346    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4347         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4348                        (match_operand:V8HF 2 "s_register_operand" "w")
4349                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4350          VCVTTQ_M_F32_F16))
4351   ]
4352   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4353   "vpst\;vcvttt.f32.f16 %q0, %q2"
4354   [(set_attr "type" "mve_move")
4355    (set_attr "length""8")])
4356
4357 ;;
4358 ;; [vdupq_m_n_f])
4359 ;;
4360 (define_insn "mve_vdupq_m_n_f<mode>"
4361   [
4362    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4363         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4364                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4365                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4366          VDUPQ_M_N_F))
4367   ]
4368   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4369   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4370   [(set_attr "type" "mve_move")
4371    (set_attr "length""8")])
4372
4373 ;;
4374 ;; [vfmaq_f])
4375 ;;
4376 (define_insn "mve_vfmaq_f<mode>"
4377   [
4378    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4379         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4380                        (match_operand:MVE_0 2 "s_register_operand" "w")
4381                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4382          VFMAQ_F))
4383   ]
4384   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4385   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4386   [(set_attr "type" "mve_move")
4387 ])
4388
4389 ;;
4390 ;; [vfmaq_n_f])
4391 ;;
4392 (define_insn "mve_vfmaq_n_f<mode>"
4393   [
4394    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4395         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4396                        (match_operand:MVE_0 2 "s_register_operand" "w")
4397                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4398          VFMAQ_N_F))
4399   ]
4400   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4401   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4402   [(set_attr "type" "mve_move")
4403 ])
4404
4405 ;;
4406 ;; [vfmasq_n_f])
4407 ;;
4408 (define_insn "mve_vfmasq_n_f<mode>"
4409   [
4410    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4411         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4412                        (match_operand:MVE_0 2 "s_register_operand" "w")
4413                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4414          VFMASQ_N_F))
4415   ]
4416   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4417   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4418   [(set_attr "type" "mve_move")
4419 ])
4420 ;;
4421 ;; [vfmsq_f])
4422 ;;
4423 (define_insn "mve_vfmsq_f<mode>"
4424   [
4425    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4426         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4427                        (match_operand:MVE_0 2 "s_register_operand" "w")
4428                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4429          VFMSQ_F))
4430   ]
4431   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4432   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4433   [(set_attr "type" "mve_move")
4434 ])
4435
4436 ;;
4437 ;; [vmaxnmaq_m_f])
4438 ;;
4439 (define_insn "mve_vmaxnmaq_m_f<mode>"
4440   [
4441    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4442         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4443                        (match_operand:MVE_0 2 "s_register_operand" "w")
4444                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4445          VMAXNMAQ_M_F))
4446   ]
4447   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4448   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4449   [(set_attr "type" "mve_move")
4450    (set_attr "length""8")])
4451 ;;
4452 ;; [vmaxnmavq_p_f])
4453 ;;
4454 (define_insn "mve_vmaxnmavq_p_f<mode>"
4455   [
4456    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4457         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4458                        (match_operand:MVE_0 2 "s_register_operand" "w")
4459                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4460          VMAXNMAVQ_P_F))
4461   ]
4462   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4463   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4464   [(set_attr "type" "mve_move")
4465    (set_attr "length""8")])
4466
4467 ;;
4468 ;; [vmaxnmvq_p_f])
4469 ;;
4470 (define_insn "mve_vmaxnmvq_p_f<mode>"
4471   [
4472    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4473         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4474                        (match_operand:MVE_0 2 "s_register_operand" "w")
4475                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4476          VMAXNMVQ_P_F))
4477   ]
4478   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4479   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4480   [(set_attr "type" "mve_move")
4481    (set_attr "length""8")])
4482 ;;
4483 ;; [vminnmaq_m_f])
4484 ;;
4485 (define_insn "mve_vminnmaq_m_f<mode>"
4486   [
4487    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4488         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4489                        (match_operand:MVE_0 2 "s_register_operand" "w")
4490                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4491          VMINNMAQ_M_F))
4492   ]
4493   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4494   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4495   [(set_attr "type" "mve_move")
4496    (set_attr "length""8")])
4497
4498 ;;
4499 ;; [vminnmavq_p_f])
4500 ;;
4501 (define_insn "mve_vminnmavq_p_f<mode>"
4502   [
4503    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4504         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4505                        (match_operand:MVE_0 2 "s_register_operand" "w")
4506                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4507          VMINNMAVQ_P_F))
4508   ]
4509   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4510   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4511   [(set_attr "type" "mve_move")
4512    (set_attr "length""8")])
4513 ;;
4514 ;; [vminnmvq_p_f])
4515 ;;
4516 (define_insn "mve_vminnmvq_p_f<mode>"
4517   [
4518    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4519         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4520                        (match_operand:MVE_0 2 "s_register_operand" "w")
4521                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4522          VMINNMVQ_P_F))
4523   ]
4524   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4525   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4526   [(set_attr "type" "mve_move")
4527    (set_attr "length""8")])
4528
4529 ;;
4530 ;; [vmlaldavaq_s, vmlaldavaq_u])
4531 ;;
4532 (define_insn "mve_vmlaldavaq_<supf><mode>"
4533   [
4534    (set (match_operand:DI 0 "s_register_operand" "=r")
4535         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4536                        (match_operand:MVE_5 2 "s_register_operand" "w")
4537                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4538          VMLALDAVAQ))
4539   ]
4540   "TARGET_HAVE_MVE"
4541   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4542   [(set_attr "type" "mve_move")
4543 ])
4544
4545 ;;
4546 ;; [vmlaldavaxq_s])
4547 ;;
4548 (define_insn "mve_vmlaldavaxq_s<mode>"
4549   [
4550    (set (match_operand:DI 0 "s_register_operand" "=r")
4551         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4552                        (match_operand:MVE_5 2 "s_register_operand" "w")
4553                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4554          VMLALDAVAXQ_S))
4555   ]
4556   "TARGET_HAVE_MVE"
4557   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4558   [(set_attr "type" "mve_move")
4559 ])
4560
4561 ;;
4562 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4563 ;;
4564 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4565   [
4566    (set (match_operand:DI 0 "s_register_operand" "=r")
4567         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4568                        (match_operand:MVE_5 2 "s_register_operand" "w")
4569                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4570          VMLALDAVQ_P))
4571   ]
4572   "TARGET_HAVE_MVE"
4573   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4574   [(set_attr "type" "mve_move")
4575    (set_attr "length""8")])
4576
4577 ;;
4578 ;; [vmlaldavxq_p_s])
4579 ;;
4580 (define_insn "mve_vmlaldavxq_p_s<mode>"
4581   [
4582    (set (match_operand:DI 0 "s_register_operand" "=r")
4583         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4584                        (match_operand:MVE_5 2 "s_register_operand" "w")
4585                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4586          VMLALDAVXQ_P_S))
4587   ]
4588   "TARGET_HAVE_MVE"
4589   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4590   [(set_attr "type" "mve_move")
4591    (set_attr "length""8")])
4592 ;;
4593 ;; [vmlsldavaq_s])
4594 ;;
4595 (define_insn "mve_vmlsldavaq_s<mode>"
4596   [
4597    (set (match_operand:DI 0 "s_register_operand" "=r")
4598         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4599                        (match_operand:MVE_5 2 "s_register_operand" "w")
4600                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4601          VMLSLDAVAQ_S))
4602   ]
4603   "TARGET_HAVE_MVE"
4604   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4605   [(set_attr "type" "mve_move")
4606 ])
4607
4608 ;;
4609 ;; [vmlsldavaxq_s])
4610 ;;
4611 (define_insn "mve_vmlsldavaxq_s<mode>"
4612   [
4613    (set (match_operand:DI 0 "s_register_operand" "=r")
4614         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4615                        (match_operand:MVE_5 2 "s_register_operand" "w")
4616                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4617          VMLSLDAVAXQ_S))
4618   ]
4619   "TARGET_HAVE_MVE"
4620   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4621   [(set_attr "type" "mve_move")
4622 ])
4623
4624 ;;
4625 ;; [vmlsldavq_p_s])
4626 ;;
4627 (define_insn "mve_vmlsldavq_p_s<mode>"
4628   [
4629    (set (match_operand:DI 0 "s_register_operand" "=r")
4630         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4631                        (match_operand:MVE_5 2 "s_register_operand" "w")
4632                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4633          VMLSLDAVQ_P_S))
4634   ]
4635   "TARGET_HAVE_MVE"
4636   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4637   [(set_attr "type" "mve_move")
4638    (set_attr "length""8")])
4639
4640 ;;
4641 ;; [vmlsldavxq_p_s])
4642 ;;
4643 (define_insn "mve_vmlsldavxq_p_s<mode>"
4644   [
4645    (set (match_operand:DI 0 "s_register_operand" "=r")
4646         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4647                        (match_operand:MVE_5 2 "s_register_operand" "w")
4648                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4649          VMLSLDAVXQ_P_S))
4650   ]
4651   "TARGET_HAVE_MVE"
4652   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4653   [(set_attr "type" "mve_move")
4654    (set_attr "length""8")])
4655 ;;
4656 ;; [vmovlbq_m_u, vmovlbq_m_s])
4657 ;;
4658 (define_insn "mve_vmovlbq_m_<supf><mode>"
4659   [
4660    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4661         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4662                        (match_operand:MVE_3 2 "s_register_operand" "w")
4663                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4664          VMOVLBQ_M))
4665   ]
4666   "TARGET_HAVE_MVE"
4667   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4668   [(set_attr "type" "mve_move")
4669    (set_attr "length""8")])
4670 ;;
4671 ;; [vmovltq_m_u, vmovltq_m_s])
4672 ;;
4673 (define_insn "mve_vmovltq_m_<supf><mode>"
4674   [
4675    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4676         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4677                        (match_operand:MVE_3 2 "s_register_operand" "w")
4678                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4679          VMOVLTQ_M))
4680   ]
4681   "TARGET_HAVE_MVE"
4682   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4683   [(set_attr "type" "mve_move")
4684    (set_attr "length""8")])
4685 ;;
4686 ;; [vmovnbq_m_u, vmovnbq_m_s])
4687 ;;
4688 (define_insn "mve_vmovnbq_m_<supf><mode>"
4689   [
4690    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4691         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4692                        (match_operand:MVE_5 2 "s_register_operand" "w")
4693                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4694          VMOVNBQ_M))
4695   ]
4696   "TARGET_HAVE_MVE"
4697   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4698   [(set_attr "type" "mve_move")
4699    (set_attr "length""8")])
4700
4701 ;;
4702 ;; [vmovntq_m_u, vmovntq_m_s])
4703 ;;
4704 (define_insn "mve_vmovntq_m_<supf><mode>"
4705   [
4706    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4707         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4708                        (match_operand:MVE_5 2 "s_register_operand" "w")
4709                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4710          VMOVNTQ_M))
4711   ]
4712   "TARGET_HAVE_MVE"
4713   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4714   [(set_attr "type" "mve_move")
4715    (set_attr "length""8")])
4716
4717 ;;
4718 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4719 ;;
4720 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4721   [
4722    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4723         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4724                        (match_operand:SI 2 "immediate_operand" "i")
4725                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4726          VMVNQ_M_N))
4727   ]
4728   "TARGET_HAVE_MVE"
4729   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4730   [(set_attr "type" "mve_move")
4731    (set_attr "length""8")])
4732 ;;
4733 ;; [vnegq_m_f])
4734 ;;
4735 (define_insn "mve_vnegq_m_f<mode>"
4736   [
4737    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4738         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4739                        (match_operand:MVE_0 2 "s_register_operand" "w")
4740                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4741          VNEGQ_M_F))
4742   ]
4743   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4744   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4745   [(set_attr "type" "mve_move")
4746    (set_attr "length""8")])
4747
4748 ;;
4749 ;; [vorrq_m_n_s, vorrq_m_n_u])
4750 ;;
4751 (define_insn "mve_vorrq_m_n_<supf><mode>"
4752   [
4753    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4754         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4755                        (match_operand:SI 2 "immediate_operand" "i")
4756                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4757          VORRQ_M_N))
4758   ]
4759   "TARGET_HAVE_MVE"
4760   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4761   [(set_attr "type" "mve_move")
4762    (set_attr "length""8")])
4763 ;;
4764 ;; [vpselq_f])
4765 ;;
4766 (define_insn "mve_vpselq_f<mode>"
4767   [
4768    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4769         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4770                        (match_operand:MVE_0 2 "s_register_operand" "w")
4771                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4772          VPSELQ_F))
4773   ]
4774   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4775   "vpsel %q0, %q1, %q2"
4776   [(set_attr "type" "mve_move")
4777 ])
4778
4779 ;;
4780 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4781 ;;
4782 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4783   [
4784    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4785         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4786                        (match_operand:MVE_5 2 "s_register_operand" "w")
4787                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4788          VQMOVNBQ_M))
4789   ]
4790   "TARGET_HAVE_MVE"
4791   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4792   [(set_attr "type" "mve_move")
4793    (set_attr "length""8")])
4794
4795 ;;
4796 ;; [vqmovntq_m_u, vqmovntq_m_s])
4797 ;;
4798 (define_insn "mve_vqmovntq_m_<supf><mode>"
4799   [
4800    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4801         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4802                        (match_operand:MVE_5 2 "s_register_operand" "w")
4803                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4804          VQMOVNTQ_M))
4805   ]
4806   "TARGET_HAVE_MVE"
4807   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4808   [(set_attr "type" "mve_move")
4809    (set_attr "length""8")])
4810
4811 ;;
4812 ;; [vqmovunbq_m_s])
4813 ;;
4814 (define_insn "mve_vqmovunbq_m_s<mode>"
4815   [
4816    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4817         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4818                        (match_operand:MVE_5 2 "s_register_operand" "w")
4819                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4820          VQMOVUNBQ_M_S))
4821   ]
4822   "TARGET_HAVE_MVE"
4823   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4824   [(set_attr "type" "mve_move")
4825    (set_attr "length""8")])
4826
4827 ;;
4828 ;; [vqmovuntq_m_s])
4829 ;;
4830 (define_insn "mve_vqmovuntq_m_s<mode>"
4831   [
4832    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4833         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4834                        (match_operand:MVE_5 2 "s_register_operand" "w")
4835                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4836          VQMOVUNTQ_M_S))
4837   ]
4838   "TARGET_HAVE_MVE"
4839   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4840   [(set_attr "type" "mve_move")
4841    (set_attr "length""8")])
4842
4843 ;;
4844 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4845 ;;
4846 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4847   [
4848    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4849         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4850                        (match_operand:MVE_5 2 "s_register_operand" "w")
4851                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4852          VQRSHRNTQ_N))
4853   ]
4854   "TARGET_HAVE_MVE"
4855   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4856   [(set_attr "type" "mve_move")
4857 ])
4858
4859 ;;
4860 ;; [vqrshruntq_n_s])
4861 ;;
4862 (define_insn "mve_vqrshruntq_n_s<mode>"
4863   [
4864    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4865         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4866                        (match_operand:MVE_5 2 "s_register_operand" "w")
4867                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4868          VQRSHRUNTQ_N_S))
4869   ]
4870   "TARGET_HAVE_MVE"
4871   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4872   [(set_attr "type" "mve_move")
4873 ])
4874
4875 ;;
4876 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4877 ;;
4878 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4879   [
4880    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4881         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4882                        (match_operand:MVE_5 2 "s_register_operand" "w")
4883                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4884          VQSHRNBQ_N))
4885   ]
4886   "TARGET_HAVE_MVE"
4887   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4888   [(set_attr "type" "mve_move")
4889 ])
4890
4891 ;;
4892 ;; [vqshrntq_n_u, vqshrntq_n_s])
4893 ;;
4894 (define_insn "mve_vqshrntq_n_<supf><mode>"
4895   [
4896    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4897         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4898                        (match_operand:MVE_5 2 "s_register_operand" "w")
4899                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4900          VQSHRNTQ_N))
4901   ]
4902   "TARGET_HAVE_MVE"
4903   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4904   [(set_attr "type" "mve_move")
4905 ])
4906
4907 ;;
4908 ;; [vqshrunbq_n_s])
4909 ;;
4910 (define_insn "mve_vqshrunbq_n_s<mode>"
4911   [
4912    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4913         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4914                        (match_operand:MVE_5 2 "s_register_operand" "w")
4915                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4916          VQSHRUNBQ_N_S))
4917   ]
4918   "TARGET_HAVE_MVE"
4919   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4920   [(set_attr "type" "mve_move")
4921 ])
4922
4923 ;;
4924 ;; [vqshruntq_n_s])
4925 ;;
4926 (define_insn "mve_vqshruntq_n_s<mode>"
4927   [
4928    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4929         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4930                        (match_operand:MVE_5 2 "s_register_operand" "w")
4931                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4932          VQSHRUNTQ_N_S))
4933   ]
4934   "TARGET_HAVE_MVE"
4935   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4936   [(set_attr "type" "mve_move")
4937 ])
4938
4939 ;;
4940 ;; [vrev32q_m_f])
4941 ;;
4942 (define_insn "mve_vrev32q_m_fv8hf"
4943   [
4944    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4945         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4946                        (match_operand:V8HF 2 "s_register_operand" "w")
4947                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4948          VREV32Q_M_F))
4949   ]
4950   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4951   "vpst\;vrev32t.16 %q0, %q2"
4952   [(set_attr "type" "mve_move")
4953    (set_attr "length""8")])
4954
4955 ;;
4956 ;; [vrev32q_m_s, vrev32q_m_u])
4957 ;;
4958 (define_insn "mve_vrev32q_m_<supf><mode>"
4959   [
4960    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4961         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4962                        (match_operand:MVE_3 2 "s_register_operand" "w")
4963                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4964          VREV32Q_M))
4965   ]
4966   "TARGET_HAVE_MVE"
4967   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4968   [(set_attr "type" "mve_move")
4969    (set_attr "length""8")])
4970
4971 ;;
4972 ;; [vrev64q_m_f])
4973 ;;
4974 (define_insn "mve_vrev64q_m_f<mode>"
4975   [
4976    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4977         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4978                        (match_operand:MVE_0 2 "s_register_operand" "w")
4979                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4980          VREV64Q_M_F))
4981   ]
4982   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4983   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4984   [(set_attr "type" "mve_move")
4985    (set_attr "length""8")])
4986
4987 ;;
4988 ;; [vrmlaldavhaxq_s])
4989 ;;
4990 (define_insn "mve_vrmlaldavhaxq_sv4si"
4991   [
4992    (set (match_operand:DI 0 "s_register_operand" "=r")
4993         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4994                        (match_operand:V4SI 2 "s_register_operand" "w")
4995                        (match_operand:V4SI 3 "s_register_operand" "w")]
4996          VRMLALDAVHAXQ_S))
4997   ]
4998   "TARGET_HAVE_MVE"
4999   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5000   [(set_attr "type" "mve_move")
5001 ])
5002
5003 ;;
5004 ;; [vrmlaldavhxq_p_s])
5005 ;;
5006 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5007   [
5008    (set (match_operand:DI 0 "s_register_operand" "=r")
5009         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5010                        (match_operand:V4SI 2 "s_register_operand" "w")
5011                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5012          VRMLALDAVHXQ_P_S))
5013   ]
5014   "TARGET_HAVE_MVE"
5015   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5016   [(set_attr "type" "mve_move")
5017    (set_attr "length""8")])
5018
5019 ;;
5020 ;; [vrmlsldavhaxq_s])
5021 ;;
5022 (define_insn "mve_vrmlsldavhaxq_sv4si"
5023   [
5024    (set (match_operand:DI 0 "s_register_operand" "=r")
5025         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5026                        (match_operand:V4SI 2 "s_register_operand" "w")
5027                        (match_operand:V4SI 3 "s_register_operand" "w")]
5028          VRMLSLDAVHAXQ_S))
5029   ]
5030   "TARGET_HAVE_MVE"
5031   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5032   [(set_attr "type" "mve_move")
5033 ])
5034
5035 ;;
5036 ;; [vrmlsldavhq_p_s])
5037 ;;
5038 (define_insn "mve_vrmlsldavhq_p_sv4si"
5039   [
5040    (set (match_operand:DI 0 "s_register_operand" "=r")
5041         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5042                        (match_operand:V4SI 2 "s_register_operand" "w")
5043                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5044          VRMLSLDAVHQ_P_S))
5045   ]
5046   "TARGET_HAVE_MVE"
5047   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5048   [(set_attr "type" "mve_move")
5049    (set_attr "length""8")])
5050
5051 ;;
5052 ;; [vrmlsldavhxq_p_s])
5053 ;;
5054 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5055   [
5056    (set (match_operand:DI 0 "s_register_operand" "=r")
5057         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5058                        (match_operand:V4SI 2 "s_register_operand" "w")
5059                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5060          VRMLSLDAVHXQ_P_S))
5061   ]
5062   "TARGET_HAVE_MVE"
5063   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5064   [(set_attr "type" "mve_move")
5065    (set_attr "length""8")])
5066
5067 ;;
5068 ;; [vrndaq_m_f])
5069 ;;
5070 (define_insn "mve_vrndaq_m_f<mode>"
5071   [
5072    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5073         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5074                        (match_operand:MVE_0 2 "s_register_operand" "w")
5075                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5076          VRNDAQ_M_F))
5077   ]
5078   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5079   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5080   [(set_attr "type" "mve_move")
5081    (set_attr "length""8")])
5082
5083 ;;
5084 ;; [vrndmq_m_f])
5085 ;;
5086 (define_insn "mve_vrndmq_m_f<mode>"
5087   [
5088    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5089         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5090                        (match_operand:MVE_0 2 "s_register_operand" "w")
5091                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5092          VRNDMQ_M_F))
5093   ]
5094   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5095   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5096   [(set_attr "type" "mve_move")
5097    (set_attr "length""8")])
5098
5099 ;;
5100 ;; [vrndnq_m_f])
5101 ;;
5102 (define_insn "mve_vrndnq_m_f<mode>"
5103   [
5104    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5105         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5106                        (match_operand:MVE_0 2 "s_register_operand" "w")
5107                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5108          VRNDNQ_M_F))
5109   ]
5110   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5111   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5112   [(set_attr "type" "mve_move")
5113    (set_attr "length""8")])
5114
5115 ;;
5116 ;; [vrndpq_m_f])
5117 ;;
5118 (define_insn "mve_vrndpq_m_f<mode>"
5119   [
5120    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5121         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5122                        (match_operand:MVE_0 2 "s_register_operand" "w")
5123                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5124          VRNDPQ_M_F))
5125   ]
5126   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5127   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5128   [(set_attr "type" "mve_move")
5129    (set_attr "length""8")])
5130
5131 ;;
5132 ;; [vrndxq_m_f])
5133 ;;
5134 (define_insn "mve_vrndxq_m_f<mode>"
5135   [
5136    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5137         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5138                        (match_operand:MVE_0 2 "s_register_operand" "w")
5139                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5140          VRNDXQ_M_F))
5141   ]
5142   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5143   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5144   [(set_attr "type" "mve_move")
5145    (set_attr "length""8")])
5146
5147 ;;
5148 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5149 ;;
5150 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5151   [
5152    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5153         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5154                        (match_operand:MVE_5 2 "s_register_operand" "w")
5155                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5156          VRSHRNBQ_N))
5157   ]
5158   "TARGET_HAVE_MVE"
5159   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5160   [(set_attr "type" "mve_move")
5161 ])
5162
5163 ;;
5164 ;; [vrshrntq_n_u, vrshrntq_n_s])
5165 ;;
5166 (define_insn "mve_vrshrntq_n_<supf><mode>"
5167   [
5168    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5169         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5170                        (match_operand:MVE_5 2 "s_register_operand" "w")
5171                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5172          VRSHRNTQ_N))
5173   ]
5174   "TARGET_HAVE_MVE"
5175   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5176   [(set_attr "type" "mve_move")
5177 ])
5178
5179 ;;
5180 ;; [vshrnbq_n_u, vshrnbq_n_s])
5181 ;;
5182 (define_insn "mve_vshrnbq_n_<supf><mode>"
5183   [
5184    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5185         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5186                        (match_operand:MVE_5 2 "s_register_operand" "w")
5187                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5188          VSHRNBQ_N))
5189   ]
5190   "TARGET_HAVE_MVE"
5191   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5192   [(set_attr "type" "mve_move")
5193 ])
5194
5195 ;;
5196 ;; [vshrntq_n_s, vshrntq_n_u])
5197 ;;
5198 (define_insn "mve_vshrntq_n_<supf><mode>"
5199   [
5200    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5201         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5202                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5203                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5204          VSHRNTQ_N))
5205   ]
5206   "TARGET_HAVE_MVE"
5207   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5208   [(set_attr "type" "mve_move")
5209 ])
5210
5211 ;;
5212 ;; [vcvtmq_m_s, vcvtmq_m_u])
5213 ;;
5214 (define_insn "mve_vcvtmq_m_<supf><mode>"
5215   [
5216    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5217         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5218                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5219                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5220          VCVTMQ_M))
5221   ]
5222   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5223   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5224   [(set_attr "type" "mve_move")
5225    (set_attr "length""8")])
5226
5227 ;;
5228 ;; [vcvtpq_m_u, vcvtpq_m_s])
5229 ;;
5230 (define_insn "mve_vcvtpq_m_<supf><mode>"
5231   [
5232    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5233         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5234                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5235                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5236          VCVTPQ_M))
5237   ]
5238   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5239   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5240   [(set_attr "type" "mve_move")
5241    (set_attr "length""8")])
5242
5243 ;;
5244 ;; [vcvtnq_m_s, vcvtnq_m_u])
5245 ;;
5246 (define_insn "mve_vcvtnq_m_<supf><mode>"
5247   [
5248    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5249         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5250                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5251                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5252          VCVTNQ_M))
5253   ]
5254   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5255   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5256   [(set_attr "type" "mve_move")
5257    (set_attr "length""8")])
5258
5259 ;;
5260 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5261 ;;
5262 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5263   [
5264    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5265         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5266                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5267                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5268                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5269          VCVTQ_M_N_FROM_F))
5270   ]
5271   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5272   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5273   [(set_attr "type" "mve_move")
5274    (set_attr "length""8")])
5275
5276 ;;
5277 ;; [vrev16q_m_u, vrev16q_m_s])
5278 ;;
5279 (define_insn "mve_vrev16q_m_<supf>v16qi"
5280   [
5281    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5282         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5283                        (match_operand:V16QI 2 "s_register_operand" "w")
5284                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5285          VREV16Q_M))
5286   ]
5287   "TARGET_HAVE_MVE"
5288   "vpst\;vrev16t.8 %q0, %q2"
5289   [(set_attr "type" "mve_move")
5290    (set_attr "length""8")])
5291
5292 ;;
5293 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5294 ;;
5295 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5296   [
5297    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5298         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5299                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5300                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5301          VCVTQ_M_FROM_F))
5302   ]
5303   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5304   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5305   [(set_attr "type" "mve_move")
5306    (set_attr "length""8")])
5307
5308 ;;
5309 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5310 ;;
5311 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5312   [
5313    (set (match_operand:DI 0 "s_register_operand" "=r")
5314         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5315                     (match_operand:V4SI 2 "s_register_operand" "w")
5316                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5317          VRMLALDAVHQ_P))
5318   ]
5319   "TARGET_HAVE_MVE"
5320   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5321   [(set_attr "type" "mve_move")
5322    (set_attr "length""8")])
5323
5324 ;;
5325 ;; [vrmlsldavhaq_s])
5326 ;;
5327 (define_insn "mve_vrmlsldavhaq_sv4si"
5328   [
5329    (set (match_operand:DI 0 "s_register_operand" "=r")
5330         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5331                     (match_operand:V4SI 2 "s_register_operand" "w")
5332                     (match_operand:V4SI 3 "s_register_operand" "w")]
5333          VRMLSLDAVHAQ_S))
5334   ]
5335   "TARGET_HAVE_MVE"
5336   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5337   [(set_attr "type" "mve_move")
5338 ])
5339
5340 ;;
5341 ;; [vabavq_p_s, vabavq_p_u])
5342 ;;
5343 (define_insn "mve_vabavq_p_<supf><mode>"
5344   [
5345    (set (match_operand:SI 0 "s_register_operand" "=r")
5346         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5347                     (match_operand:MVE_2 2 "s_register_operand" "w")
5348                     (match_operand:MVE_2 3 "s_register_operand" "w")
5349                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5350          VABAVQ_P))
5351   ]
5352   "TARGET_HAVE_MVE"
5353   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5354   [(set_attr "type" "mve_move")
5355 ])
5356
5357 ;;
5358 ;; [vqshluq_m_n_s])
5359 ;;
5360 (define_insn "mve_vqshluq_m_n_s<mode>"
5361   [
5362    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5363         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5364                        (match_operand:MVE_2 2 "s_register_operand" "w")
5365                        (match_operand:SI 3 "mve_imm_7" "Ra")
5366                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5367          VQSHLUQ_M_N_S))
5368   ]
5369   "TARGET_HAVE_MVE"
5370   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5371   [(set_attr "type" "mve_move")])
5372
5373 ;;
5374 ;; [vshlq_m_s, vshlq_m_u])
5375 ;;
5376 (define_insn "mve_vshlq_m_<supf><mode>"
5377   [
5378    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5379         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5380                        (match_operand:MVE_2 2 "s_register_operand" "w")
5381                        (match_operand:MVE_2 3 "s_register_operand" "w")
5382                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5383          VSHLQ_M))
5384   ]
5385   "TARGET_HAVE_MVE"
5386   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5387   [(set_attr "type" "mve_move")])
5388
5389 ;;
5390 ;; [vsriq_m_n_s, vsriq_m_n_u])
5391 ;;
5392 (define_insn "mve_vsriq_m_n_<supf><mode>"
5393   [
5394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5395         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5396                        (match_operand:MVE_2 2 "s_register_operand" "w")
5397                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5398                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5399          VSRIQ_M_N))
5400   ]
5401   "TARGET_HAVE_MVE"
5402   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5403   [(set_attr "type" "mve_move")])
5404
5405 ;;
5406 ;; [vsubq_m_u, vsubq_m_s])
5407 ;;
5408 (define_insn "mve_vsubq_m_<supf><mode>"
5409   [
5410    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5411         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5412                        (match_operand:MVE_2 2 "s_register_operand" "w")
5413                        (match_operand:MVE_2 3 "s_register_operand" "w")
5414                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5415          VSUBQ_M))
5416   ]
5417   "TARGET_HAVE_MVE"
5418   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5419   [(set_attr "type" "mve_move")])
5420
5421 ;;
5422 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5423 ;;
5424 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5425   [
5426    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5427         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5428                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5429                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5430                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5431          VCVTQ_M_N_TO_F))
5432   ]
5433   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5434   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5435   [(set_attr "type" "mve_move")
5436    (set_attr "length""8")])
5437 ;;
5438 ;; [vabdq_m_s, vabdq_m_u])
5439 ;;
5440 (define_insn "mve_vabdq_m_<supf><mode>"
5441   [
5442    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5443         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5444                        (match_operand:MVE_2 2 "s_register_operand" "w")
5445                        (match_operand:MVE_2 3 "s_register_operand" "w")
5446                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5447          VABDQ_M))
5448   ]
5449   "TARGET_HAVE_MVE"
5450   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5451   [(set_attr "type" "mve_move")
5452    (set_attr "length""8")])
5453
5454 ;;
5455 ;; [vaddq_m_n_s, vaddq_m_n_u])
5456 ;;
5457 (define_insn "mve_vaddq_m_n_<supf><mode>"
5458   [
5459    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5460         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5461                        (match_operand:MVE_2 2 "s_register_operand" "w")
5462                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5463                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5464          VADDQ_M_N))
5465   ]
5466   "TARGET_HAVE_MVE"
5467   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5468   [(set_attr "type" "mve_move")
5469    (set_attr "length""8")])
5470
5471 ;;
5472 ;; [vaddq_m_u, vaddq_m_s])
5473 ;;
5474 (define_insn "mve_vaddq_m_<supf><mode>"
5475   [
5476    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5477         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5478                        (match_operand:MVE_2 2 "s_register_operand" "w")
5479                        (match_operand:MVE_2 3 "s_register_operand" "w")
5480                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5481          VADDQ_M))
5482   ]
5483   "TARGET_HAVE_MVE"
5484   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5485   [(set_attr "type" "mve_move")
5486    (set_attr "length""8")])
5487
5488 ;;
5489 ;; [vandq_m_u, vandq_m_s])
5490 ;;
5491 (define_insn "mve_vandq_m_<supf><mode>"
5492   [
5493    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5494         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5495                        (match_operand:MVE_2 2 "s_register_operand" "w")
5496                        (match_operand:MVE_2 3 "s_register_operand" "w")
5497                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5498          VANDQ_M))
5499   ]
5500   "TARGET_HAVE_MVE"
5501   "vpst\;vandt %q0, %q2, %q3"
5502   [(set_attr "type" "mve_move")
5503    (set_attr "length""8")])
5504
5505 ;;
5506 ;; [vbicq_m_u, vbicq_m_s])
5507 ;;
5508 (define_insn "mve_vbicq_m_<supf><mode>"
5509   [
5510    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5511         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5512                        (match_operand:MVE_2 2 "s_register_operand" "w")
5513                        (match_operand:MVE_2 3 "s_register_operand" "w")
5514                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5515          VBICQ_M))
5516   ]
5517   "TARGET_HAVE_MVE"
5518   "vpst\;vbict %q0, %q2, %q3"
5519   [(set_attr "type" "mve_move")
5520    (set_attr "length""8")])
5521
5522 ;;
5523 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5524 ;;
5525 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5526   [
5527    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5528         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5529                        (match_operand:MVE_2 2 "s_register_operand" "w")
5530                        (match_operand:SI 3 "s_register_operand" "r")
5531                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5532          VBRSRQ_M_N))
5533   ]
5534   "TARGET_HAVE_MVE"
5535   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5536   [(set_attr "type" "mve_move")
5537    (set_attr "length""8")])
5538
5539 ;;
5540 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5541 ;;
5542 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5543   [
5544    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5545         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5546                        (match_operand:MVE_2 2 "s_register_operand" "w")
5547                        (match_operand:MVE_2 3 "s_register_operand" "w")
5548                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5549          VCADDQ_ROT270_M))
5550   ]
5551   "TARGET_HAVE_MVE"
5552   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5553   [(set_attr "type" "mve_move")
5554    (set_attr "length""8")])
5555
5556 ;;
5557 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5558 ;;
5559 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5560   [
5561    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5562         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5563                        (match_operand:MVE_2 2 "s_register_operand" "w")
5564                        (match_operand:MVE_2 3 "s_register_operand" "w")
5565                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5566          VCADDQ_ROT90_M))
5567   ]
5568   "TARGET_HAVE_MVE"
5569   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5570   [(set_attr "type" "mve_move")
5571    (set_attr "length""8")])
5572
5573 ;;
5574 ;; [veorq_m_s, veorq_m_u])
5575 ;;
5576 (define_insn "mve_veorq_m_<supf><mode>"
5577   [
5578    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5579         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5580                        (match_operand:MVE_2 2 "s_register_operand" "w")
5581                        (match_operand:MVE_2 3 "s_register_operand" "w")
5582                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5583          VEORQ_M))
5584   ]
5585   "TARGET_HAVE_MVE"
5586   "vpst\;veort %q0, %q2, %q3"
5587   [(set_attr "type" "mve_move")
5588    (set_attr "length""8")])
5589
5590 ;;
5591 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5592 ;;
5593 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5594   [
5595    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5596         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5597                        (match_operand:MVE_2 2 "s_register_operand" "w")
5598                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5599                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5600          VHADDQ_M_N))
5601   ]
5602   "TARGET_HAVE_MVE"
5603   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5604   [(set_attr "type" "mve_move")
5605    (set_attr "length""8")])
5606
5607 ;;
5608 ;; [vhaddq_m_s, vhaddq_m_u])
5609 ;;
5610 (define_insn "mve_vhaddq_m_<supf><mode>"
5611   [
5612    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5613         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5614                        (match_operand:MVE_2 2 "s_register_operand" "w")
5615                        (match_operand:MVE_2 3 "s_register_operand" "w")
5616                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5617          VHADDQ_M))
5618   ]
5619   "TARGET_HAVE_MVE"
5620   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5621   [(set_attr "type" "mve_move")
5622    (set_attr "length""8")])
5623
5624 ;;
5625 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5626 ;;
5627 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5628   [
5629    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5630         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5631                        (match_operand:MVE_2 2 "s_register_operand" "w")
5632                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5633                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5634          VHSUBQ_M_N))
5635   ]
5636   "TARGET_HAVE_MVE"
5637   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5638   [(set_attr "type" "mve_move")
5639    (set_attr "length""8")])
5640
5641 ;;
5642 ;; [vhsubq_m_s, vhsubq_m_u])
5643 ;;
5644 (define_insn "mve_vhsubq_m_<supf><mode>"
5645   [
5646    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5647         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5648                        (match_operand:MVE_2 2 "s_register_operand" "w")
5649                        (match_operand:MVE_2 3 "s_register_operand" "w")
5650                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5651          VHSUBQ_M))
5652   ]
5653   "TARGET_HAVE_MVE"
5654   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5655   [(set_attr "type" "mve_move")
5656    (set_attr "length""8")])
5657
5658 ;;
5659 ;; [vmaxq_m_s, vmaxq_m_u])
5660 ;;
5661 (define_insn "mve_vmaxq_m_<supf><mode>"
5662   [
5663    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5664         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5665                        (match_operand:MVE_2 2 "s_register_operand" "w")
5666                        (match_operand:MVE_2 3 "s_register_operand" "w")
5667                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5668          VMAXQ_M))
5669   ]
5670   "TARGET_HAVE_MVE"
5671   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5672   [(set_attr "type" "mve_move")
5673    (set_attr "length""8")])
5674
5675 ;;
5676 ;; [vminq_m_s, vminq_m_u])
5677 ;;
5678 (define_insn "mve_vminq_m_<supf><mode>"
5679   [
5680    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5681         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5682                        (match_operand:MVE_2 2 "s_register_operand" "w")
5683                        (match_operand:MVE_2 3 "s_register_operand" "w")
5684                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5685          VMINQ_M))
5686   ]
5687   "TARGET_HAVE_MVE"
5688   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5689   [(set_attr "type" "mve_move")
5690    (set_attr "length""8")])
5691
5692 ;;
5693 ;; [vmladavaq_p_u, vmladavaq_p_s])
5694 ;;
5695 (define_insn "mve_vmladavaq_p_<supf><mode>"
5696   [
5697    (set (match_operand:SI 0 "s_register_operand" "=Te")
5698         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5699                     (match_operand:MVE_2 2 "s_register_operand" "w")
5700                     (match_operand:MVE_2 3 "s_register_operand" "w")
5701                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5702          VMLADAVAQ_P))
5703   ]
5704   "TARGET_HAVE_MVE"
5705   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5706   [(set_attr "type" "mve_move")
5707    (set_attr "length""8")])
5708
5709 ;;
5710 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5711 ;;
5712 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5713   [
5714    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5715         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5716                        (match_operand:MVE_2 2 "s_register_operand" "w")
5717                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5718                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5719          VMLAQ_M_N))
5720   ]
5721   "TARGET_HAVE_MVE"
5722   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5723   [(set_attr "type" "mve_move")
5724    (set_attr "length""8")])
5725
5726 ;;
5727 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5728 ;;
5729 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5730   [
5731    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5732         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5733                        (match_operand:MVE_2 2 "s_register_operand" "w")
5734                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5735                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5736          VMLASQ_M_N))
5737   ]
5738   "TARGET_HAVE_MVE"
5739   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5740   [(set_attr "type" "mve_move")
5741    (set_attr "length""8")])
5742
5743 ;;
5744 ;; [vmulhq_m_s, vmulhq_m_u])
5745 ;;
5746 (define_insn "mve_vmulhq_m_<supf><mode>"
5747   [
5748    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5749         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5750                        (match_operand:MVE_2 2 "s_register_operand" "w")
5751                        (match_operand:MVE_2 3 "s_register_operand" "w")
5752                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5753          VMULHQ_M))
5754   ]
5755   "TARGET_HAVE_MVE"
5756   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5757   [(set_attr "type" "mve_move")
5758    (set_attr "length""8")])
5759
5760 ;;
5761 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5762 ;;
5763 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5764   [
5765    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5766         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5767                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5768                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5769                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5770          VMULLBQ_INT_M))
5771   ]
5772   "TARGET_HAVE_MVE"
5773   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5774   [(set_attr "type" "mve_move")
5775    (set_attr "length""8")])
5776
5777 ;;
5778 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5779 ;;
5780 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5781   [
5782    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5783         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5784                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5785                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5786                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5787          VMULLTQ_INT_M))
5788   ]
5789   "TARGET_HAVE_MVE"
5790   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5791   [(set_attr "type" "mve_move")
5792    (set_attr "length""8")])
5793
5794 ;;
5795 ;; [vmulq_m_n_u, vmulq_m_n_s])
5796 ;;
5797 (define_insn "mve_vmulq_m_n_<supf><mode>"
5798   [
5799    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5800         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5801                        (match_operand:MVE_2 2 "s_register_operand" "w")
5802                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5803                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5804          VMULQ_M_N))
5805   ]
5806   "TARGET_HAVE_MVE"
5807   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5808   [(set_attr "type" "mve_move")
5809    (set_attr "length""8")])
5810
5811 ;;
5812 ;; [vmulq_m_s, vmulq_m_u])
5813 ;;
5814 (define_insn "mve_vmulq_m_<supf><mode>"
5815   [
5816    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5817         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5818                        (match_operand:MVE_2 2 "s_register_operand" "w")
5819                        (match_operand:MVE_2 3 "s_register_operand" "w")
5820                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5821          VMULQ_M))
5822   ]
5823   "TARGET_HAVE_MVE"
5824   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5825   [(set_attr "type" "mve_move")
5826    (set_attr "length""8")])
5827
5828 ;;
5829 ;; [vornq_m_u, vornq_m_s])
5830 ;;
5831 (define_insn "mve_vornq_m_<supf><mode>"
5832   [
5833    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5834         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5835                        (match_operand:MVE_2 2 "s_register_operand" "w")
5836                        (match_operand:MVE_2 3 "s_register_operand" "w")
5837                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5838          VORNQ_M))
5839   ]
5840   "TARGET_HAVE_MVE"
5841   "vpst\;vornt %q0, %q2, %q3"
5842   [(set_attr "type" "mve_move")
5843    (set_attr "length""8")])
5844
5845 ;;
5846 ;; [vorrq_m_s, vorrq_m_u])
5847 ;;
5848 (define_insn "mve_vorrq_m_<supf><mode>"
5849   [
5850    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5851         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5852                        (match_operand:MVE_2 2 "s_register_operand" "w")
5853                        (match_operand:MVE_2 3 "s_register_operand" "w")
5854                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5855          VORRQ_M))
5856   ]
5857   "TARGET_HAVE_MVE"
5858   "vpst\;vorrt %q0, %q2, %q3"
5859   [(set_attr "type" "mve_move")
5860    (set_attr "length""8")])
5861
5862 ;;
5863 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5864 ;;
5865 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5866   [
5867    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5868         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5869                        (match_operand:MVE_2 2 "s_register_operand" "w")
5870                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5871                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5872          VQADDQ_M_N))
5873   ]
5874   "TARGET_HAVE_MVE"
5875   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5876   [(set_attr "type" "mve_move")
5877    (set_attr "length""8")])
5878
5879 ;;
5880 ;; [vqaddq_m_u, vqaddq_m_s])
5881 ;;
5882 (define_insn "mve_vqaddq_m_<supf><mode>"
5883   [
5884    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5885         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5886                        (match_operand:MVE_2 2 "s_register_operand" "w")
5887                        (match_operand:MVE_2 3 "s_register_operand" "w")
5888                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5889          VQADDQ_M))
5890   ]
5891   "TARGET_HAVE_MVE"
5892   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5893   [(set_attr "type" "mve_move")
5894    (set_attr "length""8")])
5895
5896 ;;
5897 ;; [vqdmlahq_m_n_s])
5898 ;;
5899 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5900   [
5901    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5902         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5903                        (match_operand:MVE_2 2 "s_register_operand" "w")
5904                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5905                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5906          VQDMLAHQ_M_N_S))
5907   ]
5908   "TARGET_HAVE_MVE"
5909   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5910   [(set_attr "type" "mve_move")
5911    (set_attr "length""8")])
5912
5913 ;;
5914 ;; [vqdmlashq_m_n_s])
5915 ;;
5916 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5917   [
5918    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5919         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5920                        (match_operand:MVE_2 2 "s_register_operand" "w")
5921                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5922                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5923          VQDMLASHQ_M_N_S))
5924   ]
5925   "TARGET_HAVE_MVE"
5926   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5927   [(set_attr "type" "mve_move")
5928    (set_attr "length""8")])
5929
5930 ;;
5931 ;; [vqrdmlahq_m_n_s])
5932 ;;
5933 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5934   [
5935    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5936         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5937                        (match_operand:MVE_2 2 "s_register_operand" "w")
5938                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5939                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5940          VQRDMLAHQ_M_N_S))
5941   ]
5942   "TARGET_HAVE_MVE"
5943   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5944   [(set_attr "type" "mve_move")
5945    (set_attr "length""8")])
5946
5947 ;;
5948 ;; [vqrdmlashq_m_n_s])
5949 ;;
5950 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5951   [
5952    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5953         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5954                        (match_operand:MVE_2 2 "s_register_operand" "w")
5955                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5956                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5957          VQRDMLASHQ_M_N_S))
5958   ]
5959   "TARGET_HAVE_MVE"
5960   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5961   [(set_attr "type" "mve_move")
5962    (set_attr "length""8")])
5963
5964 ;;
5965 ;; [vqrshlq_m_u, vqrshlq_m_s])
5966 ;;
5967 (define_insn "mve_vqrshlq_m_<supf><mode>"
5968   [
5969    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5970         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5971                        (match_operand:MVE_2 2 "s_register_operand" "w")
5972                        (match_operand:MVE_2 3 "s_register_operand" "w")
5973                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5974          VQRSHLQ_M))
5975   ]
5976   "TARGET_HAVE_MVE"
5977   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5978   [(set_attr "type" "mve_move")
5979    (set_attr "length""8")])
5980
5981 ;;
5982 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5983 ;;
5984 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5985   [
5986    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5987         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5988                        (match_operand:MVE_2 2 "s_register_operand" "w")
5989                        (match_operand:SI 3 "immediate_operand" "i")
5990                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5991          VQSHLQ_M_N))
5992   ]
5993   "TARGET_HAVE_MVE"
5994   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5995   [(set_attr "type" "mve_move")
5996    (set_attr "length""8")])
5997
5998 ;;
5999 ;; [vqshlq_m_u, vqshlq_m_s])
6000 ;;
6001 (define_insn "mve_vqshlq_m_<supf><mode>"
6002   [
6003    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6004         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6005                        (match_operand:MVE_2 2 "s_register_operand" "w")
6006                        (match_operand:MVE_2 3 "s_register_operand" "w")
6007                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6008          VQSHLQ_M))
6009   ]
6010   "TARGET_HAVE_MVE"
6011   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6012   [(set_attr "type" "mve_move")
6013    (set_attr "length""8")])
6014
6015 ;;
6016 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6017 ;;
6018 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6019   [
6020    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6021         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6022                        (match_operand:MVE_2 2 "s_register_operand" "w")
6023                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6024                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6025          VQSUBQ_M_N))
6026   ]
6027   "TARGET_HAVE_MVE"
6028   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6029   [(set_attr "type" "mve_move")
6030    (set_attr "length""8")])
6031
6032 ;;
6033 ;; [vqsubq_m_u, vqsubq_m_s])
6034 ;;
6035 (define_insn "mve_vqsubq_m_<supf><mode>"
6036   [
6037    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6038         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6039                        (match_operand:MVE_2 2 "s_register_operand" "w")
6040                        (match_operand:MVE_2 3 "s_register_operand" "w")
6041                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6042          VQSUBQ_M))
6043   ]
6044   "TARGET_HAVE_MVE"
6045   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6046   [(set_attr "type" "mve_move")
6047    (set_attr "length""8")])
6048
6049 ;;
6050 ;; [vrhaddq_m_u, vrhaddq_m_s])
6051 ;;
6052 (define_insn "mve_vrhaddq_m_<supf><mode>"
6053   [
6054    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6055         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6056                        (match_operand:MVE_2 2 "s_register_operand" "w")
6057                        (match_operand:MVE_2 3 "s_register_operand" "w")
6058                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6059          VRHADDQ_M))
6060   ]
6061   "TARGET_HAVE_MVE"
6062   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6063   [(set_attr "type" "mve_move")
6064    (set_attr "length""8")])
6065
6066 ;;
6067 ;; [vrmulhq_m_u, vrmulhq_m_s])
6068 ;;
6069 (define_insn "mve_vrmulhq_m_<supf><mode>"
6070   [
6071    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6072         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6073                        (match_operand:MVE_2 2 "s_register_operand" "w")
6074                        (match_operand:MVE_2 3 "s_register_operand" "w")
6075                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6076          VRMULHQ_M))
6077   ]
6078   "TARGET_HAVE_MVE"
6079   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6080   [(set_attr "type" "mve_move")
6081    (set_attr "length""8")])
6082
6083 ;;
6084 ;; [vrshlq_m_s, vrshlq_m_u])
6085 ;;
6086 (define_insn "mve_vrshlq_m_<supf><mode>"
6087   [
6088    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6089         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6090                        (match_operand:MVE_2 2 "s_register_operand" "w")
6091                        (match_operand:MVE_2 3 "s_register_operand" "w")
6092                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6093          VRSHLQ_M))
6094   ]
6095   "TARGET_HAVE_MVE"
6096   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6097   [(set_attr "type" "mve_move")
6098    (set_attr "length""8")])
6099
6100 ;;
6101 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6102 ;;
6103 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6104   [
6105    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6106         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6107                        (match_operand:MVE_2 2 "s_register_operand" "w")
6108                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6109                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6110          VRSHRQ_M_N))
6111   ]
6112   "TARGET_HAVE_MVE"
6113   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6114   [(set_attr "type" "mve_move")
6115    (set_attr "length""8")])
6116
6117 ;;
6118 ;; [vshlq_m_n_s, vshlq_m_n_u])
6119 ;;
6120 (define_insn "mve_vshlq_m_n_<supf><mode>"
6121   [
6122    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6123         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6124                        (match_operand:MVE_2 2 "s_register_operand" "w")
6125                        (match_operand:SI 3 "immediate_operand" "i")
6126                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6127          VSHLQ_M_N))
6128   ]
6129   "TARGET_HAVE_MVE"
6130   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6131   [(set_attr "type" "mve_move")
6132    (set_attr "length""8")])
6133
6134 ;;
6135 ;; [vshrq_m_n_s, vshrq_m_n_u])
6136 ;;
6137 (define_insn "mve_vshrq_m_n_<supf><mode>"
6138   [
6139    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6140         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6141                        (match_operand:MVE_2 2 "s_register_operand" "w")
6142                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6143                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6144          VSHRQ_M_N))
6145   ]
6146   "TARGET_HAVE_MVE"
6147   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6148   [(set_attr "type" "mve_move")
6149    (set_attr "length""8")])
6150
6151 ;;
6152 ;; [vsliq_m_n_u, vsliq_m_n_s])
6153 ;;
6154 (define_insn "mve_vsliq_m_n_<supf><mode>"
6155    [
6156    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6157        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6158                        (match_operand:MVE_2 2 "s_register_operand" "w")
6159                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6160                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6161          VSLIQ_M_N))
6162   ]
6163   "TARGET_HAVE_MVE"
6164   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6165   [(set_attr "type" "mve_move")
6166    (set_attr "length""8")])
6167
6168 ;;
6169 ;; [vsubq_m_n_s, vsubq_m_n_u])
6170 ;;
6171 (define_insn "mve_vsubq_m_n_<supf><mode>"
6172   [
6173    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6174         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6175                        (match_operand:MVE_2 2 "s_register_operand" "w")
6176                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6177                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6178          VSUBQ_M_N))
6179   ]
6180   "TARGET_HAVE_MVE"
6181   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6182   [(set_attr "type" "mve_move")
6183    (set_attr "length""8")])
6184
6185 ;;
6186 ;; [vhcaddq_rot270_m_s])
6187 ;;
6188 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6189   [
6190    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6191         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6192                        (match_operand:MVE_2 2 "s_register_operand" "w")
6193                        (match_operand:MVE_2 3 "s_register_operand" "w")
6194                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6195          VHCADDQ_ROT270_M_S))
6196   ]
6197   "TARGET_HAVE_MVE"
6198   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6199   [(set_attr "type" "mve_move")
6200    (set_attr "length""8")])
6201
6202 ;;
6203 ;; [vhcaddq_rot90_m_s])
6204 ;;
6205 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6206   [
6207    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6208         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6209                        (match_operand:MVE_2 2 "s_register_operand" "w")
6210                        (match_operand:MVE_2 3 "s_register_operand" "w")
6211                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6212          VHCADDQ_ROT90_M_S))
6213   ]
6214   "TARGET_HAVE_MVE"
6215   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6216   [(set_attr "type" "mve_move")
6217    (set_attr "length""8")])
6218
6219 ;;
6220 ;; [vmladavaxq_p_s])
6221 ;;
6222 (define_insn "mve_vmladavaxq_p_s<mode>"
6223   [
6224    (set (match_operand:SI 0 "s_register_operand" "=Te")
6225         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6226                        (match_operand:MVE_2 2 "s_register_operand" "w")
6227                        (match_operand:MVE_2 3 "s_register_operand" "w")
6228                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6229          VMLADAVAXQ_P_S))
6230   ]
6231   "TARGET_HAVE_MVE"
6232   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6233   [(set_attr "type" "mve_move")
6234    (set_attr "length""8")])
6235
6236 ;;
6237 ;; [vmlsdavaq_p_s])
6238 ;;
6239 (define_insn "mve_vmlsdavaq_p_s<mode>"
6240   [
6241    (set (match_operand:SI 0 "s_register_operand" "=Te")
6242         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6243                        (match_operand:MVE_2 2 "s_register_operand" "w")
6244                        (match_operand:MVE_2 3 "s_register_operand" "w")
6245                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6246          VMLSDAVAQ_P_S))
6247   ]
6248   "TARGET_HAVE_MVE"
6249   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6250   [(set_attr "type" "mve_move")
6251    (set_attr "length""8")])
6252
6253 ;;
6254 ;; [vmlsdavaxq_p_s])
6255 ;;
6256 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6257   [
6258    (set (match_operand:SI 0 "s_register_operand" "=Te")
6259         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6260                        (match_operand:MVE_2 2 "s_register_operand" "w")
6261                        (match_operand:MVE_2 3 "s_register_operand" "w")
6262                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6263          VMLSDAVAXQ_P_S))
6264   ]
6265   "TARGET_HAVE_MVE"
6266   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6267   [(set_attr "type" "mve_move")
6268    (set_attr "length""8")])
6269
6270 ;;
6271 ;; [vqdmladhq_m_s])
6272 ;;
6273 (define_insn "mve_vqdmladhq_m_s<mode>"
6274   [
6275    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6276         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6277                        (match_operand:MVE_2 2 "s_register_operand" "w")
6278                        (match_operand:MVE_2 3 "s_register_operand" "w")
6279                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6280          VQDMLADHQ_M_S))
6281   ]
6282   "TARGET_HAVE_MVE"
6283   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6284   [(set_attr "type" "mve_move")
6285    (set_attr "length""8")])
6286
6287 ;;
6288 ;; [vqdmladhxq_m_s])
6289 ;;
6290 (define_insn "mve_vqdmladhxq_m_s<mode>"
6291   [
6292    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6293         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6294                        (match_operand:MVE_2 2 "s_register_operand" "w")
6295                        (match_operand:MVE_2 3 "s_register_operand" "w")
6296                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6297          VQDMLADHXQ_M_S))
6298   ]
6299   "TARGET_HAVE_MVE"
6300   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6301   [(set_attr "type" "mve_move")
6302    (set_attr "length""8")])
6303
6304 ;;
6305 ;; [vqdmlsdhq_m_s])
6306 ;;
6307 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6308   [
6309    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6310         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6311                        (match_operand:MVE_2 2 "s_register_operand" "w")
6312                        (match_operand:MVE_2 3 "s_register_operand" "w")
6313                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6314          VQDMLSDHQ_M_S))
6315   ]
6316   "TARGET_HAVE_MVE"
6317   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6318   [(set_attr "type" "mve_move")
6319    (set_attr "length""8")])
6320
6321 ;;
6322 ;; [vqdmlsdhxq_m_s])
6323 ;;
6324 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6325   [
6326    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6327         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6328                        (match_operand:MVE_2 2 "s_register_operand" "w")
6329                        (match_operand:MVE_2 3 "s_register_operand" "w")
6330                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6331          VQDMLSDHXQ_M_S))
6332   ]
6333   "TARGET_HAVE_MVE"
6334   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6335   [(set_attr "type" "mve_move")
6336    (set_attr "length""8")])
6337
6338 ;;
6339 ;; [vqdmulhq_m_n_s])
6340 ;;
6341 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6342   [
6343    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6344         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6345                        (match_operand:MVE_2 2 "s_register_operand" "w")
6346                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6347                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6348          VQDMULHQ_M_N_S))
6349   ]
6350   "TARGET_HAVE_MVE"
6351   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6352   [(set_attr "type" "mve_move")
6353    (set_attr "length""8")])
6354
6355 ;;
6356 ;; [vqdmulhq_m_s])
6357 ;;
6358 (define_insn "mve_vqdmulhq_m_s<mode>"
6359   [
6360    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6361         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6362                        (match_operand:MVE_2 2 "s_register_operand" "w")
6363                        (match_operand:MVE_2 3 "s_register_operand" "w")
6364                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6365          VQDMULHQ_M_S))
6366   ]
6367   "TARGET_HAVE_MVE"
6368   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6369   [(set_attr "type" "mve_move")
6370    (set_attr "length""8")])
6371
6372 ;;
6373 ;; [vqrdmladhq_m_s])
6374 ;;
6375 (define_insn "mve_vqrdmladhq_m_s<mode>"
6376   [
6377    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6378         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6379                        (match_operand:MVE_2 2 "s_register_operand" "w")
6380                        (match_operand:MVE_2 3 "s_register_operand" "w")
6381                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6382          VQRDMLADHQ_M_S))
6383   ]
6384   "TARGET_HAVE_MVE"
6385   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6386   [(set_attr "type" "mve_move")
6387    (set_attr "length""8")])
6388
6389 ;;
6390 ;; [vqrdmladhxq_m_s])
6391 ;;
6392 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6393   [
6394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6395         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6396                        (match_operand:MVE_2 2 "s_register_operand" "w")
6397                        (match_operand:MVE_2 3 "s_register_operand" "w")
6398                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6399          VQRDMLADHXQ_M_S))
6400   ]
6401   "TARGET_HAVE_MVE"
6402   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6403   [(set_attr "type" "mve_move")
6404    (set_attr "length""8")])
6405
6406 ;;
6407 ;; [vqrdmlsdhq_m_s])
6408 ;;
6409 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6410   [
6411    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6412         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6413                        (match_operand:MVE_2 2 "s_register_operand" "w")
6414                        (match_operand:MVE_2 3 "s_register_operand" "w")
6415                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6416          VQRDMLSDHQ_M_S))
6417   ]
6418   "TARGET_HAVE_MVE"
6419   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6420   [(set_attr "type" "mve_move")
6421    (set_attr "length""8")])
6422
6423 ;;
6424 ;; [vqrdmlsdhxq_m_s])
6425 ;;
6426 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6427   [
6428    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6429         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6430                        (match_operand:MVE_2 2 "s_register_operand" "w")
6431                        (match_operand:MVE_2 3 "s_register_operand" "w")
6432                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6433          VQRDMLSDHXQ_M_S))
6434   ]
6435   "TARGET_HAVE_MVE"
6436   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6437   [(set_attr "type" "mve_move")
6438    (set_attr "length""8")])
6439
6440 ;;
6441 ;; [vqrdmulhq_m_n_s])
6442 ;;
6443 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6444   [
6445    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6446         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6447                        (match_operand:MVE_2 2 "s_register_operand" "w")
6448                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6449                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6450          VQRDMULHQ_M_N_S))
6451   ]
6452   "TARGET_HAVE_MVE"
6453   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6454   [(set_attr "type" "mve_move")
6455    (set_attr "length""8")])
6456
6457 ;;
6458 ;; [vqrdmulhq_m_s])
6459 ;;
6460 (define_insn "mve_vqrdmulhq_m_s<mode>"
6461   [
6462    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6463         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6464                        (match_operand:MVE_2 2 "s_register_operand" "w")
6465                        (match_operand:MVE_2 3 "s_register_operand" "w")
6466                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6467          VQRDMULHQ_M_S))
6468   ]
6469   "TARGET_HAVE_MVE"
6470   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6471   [(set_attr "type" "mve_move")
6472    (set_attr "length""8")])
6473
6474 ;;
6475 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6476 ;;
6477 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6478   [
6479    (set (match_operand:DI 0 "s_register_operand" "=r")
6480         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6481                        (match_operand:MVE_5 2 "s_register_operand" "w")
6482                        (match_operand:MVE_5 3 "s_register_operand" "w")
6483                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6484          VMLALDAVAQ_P))
6485   ]
6486   "TARGET_HAVE_MVE"
6487   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6488   [(set_attr "type" "mve_move")
6489    (set_attr "length""8")])
6490
6491 ;;
6492 ;; [vmlaldavaxq_p_s])
6493 ;;
6494 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6495   [
6496    (set (match_operand:DI 0 "s_register_operand" "=r")
6497         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6498                        (match_operand:MVE_5 2 "s_register_operand" "w")
6499                        (match_operand:MVE_5 3 "s_register_operand" "w")
6500                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6501          VMLALDAVAXQ_P))
6502   ]
6503   "TARGET_HAVE_MVE"
6504   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6505   [(set_attr "type" "mve_move")
6506    (set_attr "length""8")])
6507
6508 ;;
6509 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6510 ;;
6511 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6512   [
6513    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6514         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6515                        (match_operand:MVE_5 2 "s_register_operand" "w")
6516                        (match_operand:SI 3 "mve_imm_8" "Rb")
6517                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6518          VQRSHRNBQ_M_N))
6519   ]
6520   "TARGET_HAVE_MVE"
6521   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6522   [(set_attr "type" "mve_move")
6523    (set_attr "length""8")])
6524
6525 ;;
6526 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6527 ;;
6528 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6529   [
6530    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6531         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6532                        (match_operand:MVE_5 2 "s_register_operand" "w")
6533                        (match_operand:SI 3 "mve_imm_8" "Rb")
6534                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6535          VQRSHRNTQ_M_N))
6536   ]
6537   "TARGET_HAVE_MVE"
6538   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6539   [(set_attr "type" "mve_move")
6540    (set_attr "length""8")])
6541
6542 ;;
6543 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6544 ;;
6545 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6546   [
6547    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6548         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6549                        (match_operand:MVE_5 2 "s_register_operand" "w")
6550                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6551                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6552          VQSHRNBQ_M_N))
6553   ]
6554   "TARGET_HAVE_MVE"
6555   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6556   [(set_attr "type" "mve_move")
6557    (set_attr "length""8")])
6558
6559 ;;
6560 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6561 ;;
6562 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6563   [
6564    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6565         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6566                        (match_operand:MVE_5 2 "s_register_operand" "w")
6567                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6568                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6569          VQSHRNTQ_M_N))
6570   ]
6571   "TARGET_HAVE_MVE"
6572   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6573   [(set_attr "type" "mve_move")
6574    (set_attr "length""8")])
6575
6576 ;;
6577 ;; [vrmlaldavhaq_p_s])
6578 ;;
6579 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6580   [
6581    (set (match_operand:DI 0 "s_register_operand" "=r")
6582         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6583                        (match_operand:V4SI 2 "s_register_operand" "w")
6584                        (match_operand:V4SI 3 "s_register_operand" "w")
6585                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6586          VRMLALDAVHAQ_P_S))
6587   ]
6588   "TARGET_HAVE_MVE"
6589   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6590   [(set_attr "type" "mve_move")
6591    (set_attr "length""8")])
6592
6593 ;;
6594 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6595 ;;
6596 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6597   [
6598    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6599         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6600                        (match_operand:MVE_5 2 "s_register_operand" "w")
6601                        (match_operand:SI 3 "mve_imm_8" "Rb")
6602                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6603          VRSHRNBQ_M_N))
6604   ]
6605   "TARGET_HAVE_MVE"
6606   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6607   [(set_attr "type" "mve_move")
6608    (set_attr "length""8")])
6609
6610 ;;
6611 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6612 ;;
6613 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6614   [
6615    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6616         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6617                        (match_operand:MVE_5 2 "s_register_operand" "w")
6618                        (match_operand:SI 3 "mve_imm_8" "Rb")
6619                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6620          VRSHRNTQ_M_N))
6621   ]
6622   "TARGET_HAVE_MVE"
6623   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6624   [(set_attr "type" "mve_move")
6625    (set_attr "length""8")])
6626
6627 ;;
6628 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6629 ;;
6630 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6631   [
6632    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6633         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6634                        (match_operand:MVE_3 2 "s_register_operand" "w")
6635                        (match_operand:SI 3 "immediate_operand" "i")
6636                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6637          VSHLLBQ_M_N))
6638   ]
6639   "TARGET_HAVE_MVE"
6640   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6641   [(set_attr "type" "mve_move")
6642    (set_attr "length""8")])
6643
6644 ;;
6645 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6646 ;;
6647 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6648   [
6649    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6650         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6651                        (match_operand:MVE_3 2 "s_register_operand" "w")
6652                        (match_operand:SI 3 "immediate_operand" "i")
6653                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6654          VSHLLTQ_M_N))
6655   ]
6656   "TARGET_HAVE_MVE"
6657   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6658   [(set_attr "type" "mve_move")
6659    (set_attr "length""8")])
6660
6661 ;;
6662 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6663 ;;
6664 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6665   [
6666    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6667         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6668                        (match_operand:MVE_5 2 "s_register_operand" "w")
6669                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6670                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6671          VSHRNBQ_M_N))
6672   ]
6673   "TARGET_HAVE_MVE"
6674   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6675   [(set_attr "type" "mve_move")
6676    (set_attr "length""8")])
6677
6678 ;;
6679 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6680 ;;
6681 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6682   [
6683    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6684         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6685                        (match_operand:MVE_5 2 "s_register_operand" "w")
6686                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6687                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6688          VSHRNTQ_M_N))
6689   ]
6690   "TARGET_HAVE_MVE"
6691   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6692   [(set_attr "type" "mve_move")
6693    (set_attr "length""8")])
6694
6695 ;;
6696 ;; [vmlsldavaq_p_s])
6697 ;;
6698 (define_insn "mve_vmlsldavaq_p_s<mode>"
6699   [
6700    (set (match_operand:DI 0 "s_register_operand" "=r")
6701         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6702                        (match_operand:MVE_5 2 "s_register_operand" "w")
6703                        (match_operand:MVE_5 3 "s_register_operand" "w")
6704                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6705          VMLSLDAVAQ_P_S))
6706   ]
6707   "TARGET_HAVE_MVE"
6708   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6709   [(set_attr "type" "mve_move")
6710    (set_attr "length""8")])
6711
6712 ;;
6713 ;; [vmlsldavaxq_p_s])
6714 ;;
6715 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6716   [
6717    (set (match_operand:DI 0 "s_register_operand" "=r")
6718         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6719                        (match_operand:MVE_5 2 "s_register_operand" "w")
6720                        (match_operand:MVE_5 3 "s_register_operand" "w")
6721                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6722          VMLSLDAVAXQ_P_S))
6723   ]
6724   "TARGET_HAVE_MVE"
6725   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6726   [(set_attr "type" "mve_move")
6727    (set_attr "length""8")])
6728
6729 ;;
6730 ;; [vmullbq_poly_m_p])
6731 ;;
6732 (define_insn "mve_vmullbq_poly_m_p<mode>"
6733   [
6734    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6735         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6736                        (match_operand:MVE_3 2 "s_register_operand" "w")
6737                        (match_operand:MVE_3 3 "s_register_operand" "w")
6738                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6739          VMULLBQ_POLY_M_P))
6740   ]
6741   "TARGET_HAVE_MVE"
6742   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6743   [(set_attr "type" "mve_move")
6744    (set_attr "length""8")])
6745
6746 ;;
6747 ;; [vmulltq_poly_m_p])
6748 ;;
6749 (define_insn "mve_vmulltq_poly_m_p<mode>"
6750   [
6751    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6752         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6753                        (match_operand:MVE_3 2 "s_register_operand" "w")
6754                        (match_operand:MVE_3 3 "s_register_operand" "w")
6755                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6756          VMULLTQ_POLY_M_P))
6757   ]
6758   "TARGET_HAVE_MVE"
6759   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6760   [(set_attr "type" "mve_move")
6761    (set_attr "length""8")])
6762
6763 ;;
6764 ;; [vqdmullbq_m_n_s])
6765 ;;
6766 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6767   [
6768    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6769         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6770                        (match_operand:MVE_5 2 "s_register_operand" "w")
6771                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6772                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6773          VQDMULLBQ_M_N_S))
6774   ]
6775   "TARGET_HAVE_MVE"
6776   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6777   [(set_attr "type" "mve_move")
6778    (set_attr "length""8")])
6779
6780 ;;
6781 ;; [vqdmullbq_m_s])
6782 ;;
6783 (define_insn "mve_vqdmullbq_m_s<mode>"
6784   [
6785    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6786         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6787                        (match_operand:MVE_5 2 "s_register_operand" "w")
6788                        (match_operand:MVE_5 3 "s_register_operand" "w")
6789                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6790          VQDMULLBQ_M_S))
6791   ]
6792   "TARGET_HAVE_MVE"
6793   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6794   [(set_attr "type" "mve_move")
6795    (set_attr "length""8")])
6796
6797 ;;
6798 ;; [vqdmulltq_m_n_s])
6799 ;;
6800 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6801   [
6802    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6803         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6804                        (match_operand:MVE_5 2 "s_register_operand" "w")
6805                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6806                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6807          VQDMULLTQ_M_N_S))
6808   ]
6809   "TARGET_HAVE_MVE"
6810   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6811   [(set_attr "type" "mve_move")
6812    (set_attr "length""8")])
6813
6814 ;;
6815 ;; [vqdmulltq_m_s])
6816 ;;
6817 (define_insn "mve_vqdmulltq_m_s<mode>"
6818   [
6819    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6820         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6821                        (match_operand:MVE_5 2 "s_register_operand" "w")
6822                        (match_operand:MVE_5 3 "s_register_operand" "w")
6823                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6824          VQDMULLTQ_M_S))
6825   ]
6826   "TARGET_HAVE_MVE"
6827   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6828   [(set_attr "type" "mve_move")
6829    (set_attr "length""8")])
6830
6831 ;;
6832 ;; [vqrshrunbq_m_n_s])
6833 ;;
6834 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6835   [
6836    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6837         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6838                        (match_operand:MVE_5 2 "s_register_operand" "w")
6839                        (match_operand:SI 3 "mve_imm_8" "Rb")
6840                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6841          VQRSHRUNBQ_M_N_S))
6842   ]
6843   "TARGET_HAVE_MVE"
6844   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6845   [(set_attr "type" "mve_move")
6846    (set_attr "length""8")])
6847
6848 ;;
6849 ;; [vqrshruntq_m_n_s])
6850 ;;
6851 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6852   [
6853    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6854         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6855                        (match_operand:MVE_5 2 "s_register_operand" "w")
6856                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6857                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6858          VQRSHRUNTQ_M_N_S))
6859   ]
6860   "TARGET_HAVE_MVE"
6861   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6862   [(set_attr "type" "mve_move")
6863    (set_attr "length""8")])
6864
6865 ;;
6866 ;; [vqshrunbq_m_n_s])
6867 ;;
6868 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6869   [
6870    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6871         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6872                        (match_operand:MVE_5 2 "s_register_operand" "w")
6873                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6874                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6875          VQSHRUNBQ_M_N_S))
6876   ]
6877   "TARGET_HAVE_MVE"
6878   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6879   [(set_attr "type" "mve_move")
6880    (set_attr "length""8")])
6881
6882 ;;
6883 ;; [vqshruntq_m_n_s])
6884 ;;
6885 (define_insn "mve_vqshruntq_m_n_s<mode>"
6886   [
6887    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6888         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6889                        (match_operand:MVE_5 2 "s_register_operand" "w")
6890                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6891                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6892          VQSHRUNTQ_M_N_S))
6893   ]
6894   "TARGET_HAVE_MVE"
6895   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6896   [(set_attr "type" "mve_move")
6897    (set_attr "length""8")])
6898
6899 ;;
6900 ;; [vrmlaldavhaq_p_u])
6901 ;;
6902 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6903   [
6904    (set (match_operand:DI 0 "s_register_operand" "=r")
6905         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6906                        (match_operand:V4SI 2 "s_register_operand" "w")
6907                        (match_operand:V4SI 3 "s_register_operand" "w")
6908                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6909          VRMLALDAVHAQ_P_U))
6910   ]
6911   "TARGET_HAVE_MVE"
6912   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6913   [(set_attr "type" "mve_move")
6914    (set_attr "length""8")])
6915
6916 ;;
6917 ;; [vrmlaldavhaxq_p_s])
6918 ;;
6919 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6920   [
6921    (set (match_operand:DI 0 "s_register_operand" "=r")
6922         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6923                        (match_operand:V4SI 2 "s_register_operand" "w")
6924                        (match_operand:V4SI 3 "s_register_operand" "w")
6925                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6926          VRMLALDAVHAXQ_P_S))
6927   ]
6928   "TARGET_HAVE_MVE"
6929   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6930   [(set_attr "type" "mve_move")
6931    (set_attr "length""8")])
6932
6933 ;;
6934 ;; [vrmlsldavhaq_p_s])
6935 ;;
6936 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6937   [
6938    (set (match_operand:DI 0 "s_register_operand" "=r")
6939         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6940                        (match_operand:V4SI 2 "s_register_operand" "w")
6941                        (match_operand:V4SI 3 "s_register_operand" "w")
6942                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6943          VRMLSLDAVHAQ_P_S))
6944   ]
6945   "TARGET_HAVE_MVE"
6946   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6947   [(set_attr "type" "mve_move")
6948    (set_attr "length""8")])
6949
6950 ;;
6951 ;; [vrmlsldavhaxq_p_s])
6952 ;;
6953 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6954   [
6955    (set (match_operand:DI 0 "s_register_operand" "=r")
6956         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6957                        (match_operand:V4SI 2 "s_register_operand" "w")
6958                        (match_operand:V4SI 3 "s_register_operand" "w")
6959                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6960          VRMLSLDAVHAXQ_P_S))
6961   ]
6962   "TARGET_HAVE_MVE"
6963   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6964   [(set_attr "type" "mve_move")
6965    (set_attr "length""8")])
6966 ;;
6967 ;; [vabdq_m_f])
6968 ;;
6969 (define_insn "mve_vabdq_m_f<mode>"
6970   [
6971    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6972         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6973                        (match_operand:MVE_0 2 "s_register_operand" "w")
6974                        (match_operand:MVE_0 3 "s_register_operand" "w")
6975                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6976          VABDQ_M_F))
6977   ]
6978   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6979   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6980   [(set_attr "type" "mve_move")
6981    (set_attr "length""8")])
6982
6983 ;;
6984 ;; [vaddq_m_f])
6985 ;;
6986 (define_insn "mve_vaddq_m_f<mode>"
6987   [
6988    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6989         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6990                        (match_operand:MVE_0 2 "s_register_operand" "w")
6991                        (match_operand:MVE_0 3 "s_register_operand" "w")
6992                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6993          VADDQ_M_F))
6994   ]
6995   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6996   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6997   [(set_attr "type" "mve_move")
6998    (set_attr "length""8")])
6999
7000 ;;
7001 ;; [vaddq_m_n_f])
7002 ;;
7003 (define_insn "mve_vaddq_m_n_f<mode>"
7004   [
7005    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7006         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7007                        (match_operand:MVE_0 2 "s_register_operand" "w")
7008                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7009                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7010          VADDQ_M_N_F))
7011   ]
7012   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7013   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7014   [(set_attr "type" "mve_move")
7015    (set_attr "length""8")])
7016
7017 ;;
7018 ;; [vandq_m_f])
7019 ;;
7020 (define_insn "mve_vandq_m_f<mode>"
7021   [
7022    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7023         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7024                        (match_operand:MVE_0 2 "s_register_operand" "w")
7025                        (match_operand:MVE_0 3 "s_register_operand" "w")
7026                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7027          VANDQ_M_F))
7028   ]
7029   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7030   "vpst\;vandt %q0, %q2, %q3"
7031   [(set_attr "type" "mve_move")
7032    (set_attr "length""8")])
7033
7034 ;;
7035 ;; [vbicq_m_f])
7036 ;;
7037 (define_insn "mve_vbicq_m_f<mode>"
7038   [
7039    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7040         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7041                        (match_operand:MVE_0 2 "s_register_operand" "w")
7042                        (match_operand:MVE_0 3 "s_register_operand" "w")
7043                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7044          VBICQ_M_F))
7045   ]
7046   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7047   "vpst\;vbict %q0, %q2, %q3"
7048   [(set_attr "type" "mve_move")
7049    (set_attr "length""8")])
7050
7051 ;;
7052 ;; [vbrsrq_m_n_f])
7053 ;;
7054 (define_insn "mve_vbrsrq_m_n_f<mode>"
7055   [
7056    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7057         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7058                        (match_operand:MVE_0 2 "s_register_operand" "w")
7059                        (match_operand:SI 3 "s_register_operand" "r")
7060                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7061          VBRSRQ_M_N_F))
7062   ]
7063   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7064   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7065   [(set_attr "type" "mve_move")
7066    (set_attr "length""8")])
7067
7068 ;;
7069 ;; [vcaddq_rot270_m_f])
7070 ;;
7071 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7072   [
7073    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7074         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7075                        (match_operand:MVE_0 2 "s_register_operand" "w")
7076                        (match_operand:MVE_0 3 "s_register_operand" "w")
7077                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7078          VCADDQ_ROT270_M_F))
7079   ]
7080   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7081   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7082   [(set_attr "type" "mve_move")
7083    (set_attr "length""8")])
7084
7085 ;;
7086 ;; [vcaddq_rot90_m_f])
7087 ;;
7088 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7089   [
7090    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7091         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7092                        (match_operand:MVE_0 2 "s_register_operand" "w")
7093                        (match_operand:MVE_0 3 "s_register_operand" "w")
7094                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7095          VCADDQ_ROT90_M_F))
7096   ]
7097   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7098   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7099   [(set_attr "type" "mve_move")
7100    (set_attr "length""8")])
7101
7102 ;;
7103 ;; [vcmlaq_m_f])
7104 ;;
7105 (define_insn "mve_vcmlaq_m_f<mode>"
7106   [
7107    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7108         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7109                        (match_operand:MVE_0 2 "s_register_operand" "w")
7110                        (match_operand:MVE_0 3 "s_register_operand" "w")
7111                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7112          VCMLAQ_M_F))
7113   ]
7114   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7115   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7116   [(set_attr "type" "mve_move")
7117    (set_attr "length""8")])
7118
7119 ;;
7120 ;; [vcmlaq_rot180_m_f])
7121 ;;
7122 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7123   [
7124    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7125         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7126                        (match_operand:MVE_0 2 "s_register_operand" "w")
7127                        (match_operand:MVE_0 3 "s_register_operand" "w")
7128                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7129          VCMLAQ_ROT180_M_F))
7130   ]
7131   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7132   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7133   [(set_attr "type" "mve_move")
7134    (set_attr "length""8")])
7135
7136 ;;
7137 ;; [vcmlaq_rot270_m_f])
7138 ;;
7139 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7140   [
7141    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7142         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7143                        (match_operand:MVE_0 2 "s_register_operand" "w")
7144                        (match_operand:MVE_0 3 "s_register_operand" "w")
7145                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7146          VCMLAQ_ROT270_M_F))
7147   ]
7148   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7149   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7150   [(set_attr "type" "mve_move")
7151    (set_attr "length""8")])
7152
7153 ;;
7154 ;; [vcmlaq_rot90_m_f])
7155 ;;
7156 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7157   [
7158    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7159         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7160                        (match_operand:MVE_0 2 "s_register_operand" "w")
7161                        (match_operand:MVE_0 3 "s_register_operand" "w")
7162                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7163          VCMLAQ_ROT90_M_F))
7164   ]
7165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7166   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7167   [(set_attr "type" "mve_move")
7168    (set_attr "length""8")])
7169
7170 ;;
7171 ;; [vcmulq_m_f])
7172 ;;
7173 (define_insn "mve_vcmulq_m_f<mode>"
7174   [
7175    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7177                        (match_operand:MVE_0 2 "s_register_operand" "w")
7178                        (match_operand:MVE_0 3 "s_register_operand" "w")
7179                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7180          VCMULQ_M_F))
7181   ]
7182   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7183   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7184   [(set_attr "type" "mve_move")
7185    (set_attr "length""8")])
7186
7187 ;;
7188 ;; [vcmulq_rot180_m_f])
7189 ;;
7190 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7191   [
7192    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7193         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7194                        (match_operand:MVE_0 2 "s_register_operand" "w")
7195                        (match_operand:MVE_0 3 "s_register_operand" "w")
7196                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7197          VCMULQ_ROT180_M_F))
7198   ]
7199   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7200   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7201   [(set_attr "type" "mve_move")
7202    (set_attr "length""8")])
7203
7204 ;;
7205 ;; [vcmulq_rot270_m_f])
7206 ;;
7207 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7208   [
7209    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7210         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7211                        (match_operand:MVE_0 2 "s_register_operand" "w")
7212                        (match_operand:MVE_0 3 "s_register_operand" "w")
7213                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7214          VCMULQ_ROT270_M_F))
7215   ]
7216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7217   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7218   [(set_attr "type" "mve_move")
7219    (set_attr "length""8")])
7220
7221 ;;
7222 ;; [vcmulq_rot90_m_f])
7223 ;;
7224 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7225   [
7226    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7227         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7228                        (match_operand:MVE_0 2 "s_register_operand" "w")
7229                        (match_operand:MVE_0 3 "s_register_operand" "w")
7230                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7231          VCMULQ_ROT90_M_F))
7232   ]
7233   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7234   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7235   [(set_attr "type" "mve_move")
7236    (set_attr "length""8")])
7237
7238 ;;
7239 ;; [veorq_m_f])
7240 ;;
7241 (define_insn "mve_veorq_m_f<mode>"
7242   [
7243    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7244         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7245                        (match_operand:MVE_0 2 "s_register_operand" "w")
7246                        (match_operand:MVE_0 3 "s_register_operand" "w")
7247                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7248          VEORQ_M_F))
7249   ]
7250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7251   "vpst\;veort %q0, %q2, %q3"
7252   [(set_attr "type" "mve_move")
7253    (set_attr "length""8")])
7254
7255 ;;
7256 ;; [vfmaq_m_f])
7257 ;;
7258 (define_insn "mve_vfmaq_m_f<mode>"
7259   [
7260    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7261         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7262                        (match_operand:MVE_0 2 "s_register_operand" "w")
7263                        (match_operand:MVE_0 3 "s_register_operand" "w")
7264                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7265          VFMAQ_M_F))
7266   ]
7267   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7268   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7269   [(set_attr "type" "mve_move")
7270    (set_attr "length""8")])
7271
7272 ;;
7273 ;; [vfmaq_m_n_f])
7274 ;;
7275 (define_insn "mve_vfmaq_m_n_f<mode>"
7276   [
7277    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7278         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7279                        (match_operand:MVE_0 2 "s_register_operand" "w")
7280                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7281                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7282          VFMAQ_M_N_F))
7283   ]
7284   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7285   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7286   [(set_attr "type" "mve_move")
7287    (set_attr "length""8")])
7288
7289 ;;
7290 ;; [vfmasq_m_n_f])
7291 ;;
7292 (define_insn "mve_vfmasq_m_n_f<mode>"
7293   [
7294    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7295         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7296                        (match_operand:MVE_0 2 "s_register_operand" "w")
7297                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7298                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7299          VFMASQ_M_N_F))
7300   ]
7301   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7302   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7303   [(set_attr "type" "mve_move")
7304    (set_attr "length""8")])
7305
7306 ;;
7307 ;; [vfmsq_m_f])
7308 ;;
7309 (define_insn "mve_vfmsq_m_f<mode>"
7310   [
7311    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7312         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7313                        (match_operand:MVE_0 2 "s_register_operand" "w")
7314                        (match_operand:MVE_0 3 "s_register_operand" "w")
7315                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7316          VFMSQ_M_F))
7317   ]
7318   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7319   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7320   [(set_attr "type" "mve_move")
7321    (set_attr "length""8")])
7322
7323 ;;
7324 ;; [vmaxnmq_m_f])
7325 ;;
7326 (define_insn "mve_vmaxnmq_m_f<mode>"
7327   [
7328    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7329         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7330                        (match_operand:MVE_0 2 "s_register_operand" "w")
7331                        (match_operand:MVE_0 3 "s_register_operand" "w")
7332                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7333          VMAXNMQ_M_F))
7334   ]
7335   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7336   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7337   [(set_attr "type" "mve_move")
7338    (set_attr "length""8")])
7339
7340 ;;
7341 ;; [vminnmq_m_f])
7342 ;;
7343 (define_insn "mve_vminnmq_m_f<mode>"
7344   [
7345    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7346         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7347                        (match_operand:MVE_0 2 "s_register_operand" "w")
7348                        (match_operand:MVE_0 3 "s_register_operand" "w")
7349                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7350          VMINNMQ_M_F))
7351   ]
7352   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7353   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7354   [(set_attr "type" "mve_move")
7355    (set_attr "length""8")])
7356
7357 ;;
7358 ;; [vmulq_m_f])
7359 ;;
7360 (define_insn "mve_vmulq_m_f<mode>"
7361   [
7362    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7363         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7364                        (match_operand:MVE_0 2 "s_register_operand" "w")
7365                        (match_operand:MVE_0 3 "s_register_operand" "w")
7366                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7367          VMULQ_M_F))
7368   ]
7369   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7370   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7371   [(set_attr "type" "mve_move")
7372    (set_attr "length""8")])
7373
7374 ;;
7375 ;; [vmulq_m_n_f])
7376 ;;
7377 (define_insn "mve_vmulq_m_n_f<mode>"
7378   [
7379    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7380         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7381                        (match_operand:MVE_0 2 "s_register_operand" "w")
7382                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7383                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7384          VMULQ_M_N_F))
7385   ]
7386   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7387   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7388   [(set_attr "type" "mve_move")
7389    (set_attr "length""8")])
7390
7391 ;;
7392 ;; [vornq_m_f])
7393 ;;
7394 (define_insn "mve_vornq_m_f<mode>"
7395   [
7396    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7397         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7398                        (match_operand:MVE_0 2 "s_register_operand" "w")
7399                        (match_operand:MVE_0 3 "s_register_operand" "w")
7400                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7401          VORNQ_M_F))
7402   ]
7403   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7404   "vpst\;vornt %q0, %q2, %q3"
7405   [(set_attr "type" "mve_move")
7406    (set_attr "length""8")])
7407
7408 ;;
7409 ;; [vorrq_m_f])
7410 ;;
7411 (define_insn "mve_vorrq_m_f<mode>"
7412   [
7413    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7414         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7415                        (match_operand:MVE_0 2 "s_register_operand" "w")
7416                        (match_operand:MVE_0 3 "s_register_operand" "w")
7417                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7418          VORRQ_M_F))
7419   ]
7420   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7421   "vpst\;vorrt %q0, %q2, %q3"
7422   [(set_attr "type" "mve_move")
7423    (set_attr "length""8")])
7424
7425 ;;
7426 ;; [vsubq_m_f])
7427 ;;
7428 (define_insn "mve_vsubq_m_f<mode>"
7429   [
7430    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7431         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7432                        (match_operand:MVE_0 2 "s_register_operand" "w")
7433                        (match_operand:MVE_0 3 "s_register_operand" "w")
7434                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7435          VSUBQ_M_F))
7436   ]
7437   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7438   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7439   [(set_attr "type" "mve_move")
7440    (set_attr "length""8")])
7441
7442 ;;
7443 ;; [vsubq_m_n_f])
7444 ;;
7445 (define_insn "mve_vsubq_m_n_f<mode>"
7446   [
7447    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7448         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7449                        (match_operand:MVE_0 2 "s_register_operand" "w")
7450                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7451                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7452          VSUBQ_M_N_F))
7453   ]
7454   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7455   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7456   [(set_attr "type" "mve_move")
7457    (set_attr "length""8")])
7458
7459 ;;
7460 ;; [vstrbq_s vstrbq_u]
7461 ;;
7462 (define_insn "mve_vstrbq_<supf><mode>"
7463   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7464         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7465          VSTRBQ))
7466   ]
7467   "TARGET_HAVE_MVE"
7468 {
7469    rtx ops[2];
7470    int regno = REGNO (operands[1]);
7471    ops[1] = gen_rtx_REG (TImode, regno);
7472    ops[0]  = operands[0];
7473    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7474    return "";
7475 }
7476   [(set_attr "length" "4")])
7477
7478 ;;
7479 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7480 ;;
7481 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7482   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7483    (match_operand:MVE_2 1 "s_register_operand")
7484    (match_operand:MVE_2 2 "s_register_operand")
7485    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7486   "TARGET_HAVE_MVE"
7487 {
7488   rtx ind = XEXP (operands[0], 0);
7489   gcc_assert (REG_P (ind));
7490   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7491                                                               operands[2]));
7492   DONE;
7493 })
7494
7495 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7496   [(set (mem:BLK (scratch))
7497         (unspec:BLK
7498           [(match_operand:SI 0 "register_operand" "r")
7499            (match_operand:MVE_2 1 "s_register_operand" "w")
7500            (match_operand:MVE_2 2 "s_register_operand" "w")]
7501           VSTRBSOQ))]
7502   "TARGET_HAVE_MVE"
7503   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7504   [(set_attr "length" "4")])
7505
7506 ;;
7507 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7508 ;;
7509 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7510   [(set (mem:BLK (scratch))
7511         (unspec:BLK
7512                 [(match_operand:V4SI 0 "s_register_operand" "w")
7513                  (match_operand:SI 1 "immediate_operand" "i")
7514                  (match_operand:V4SI 2 "s_register_operand" "w")]
7515          VSTRWSBQ))
7516   ]
7517   "TARGET_HAVE_MVE"
7518 {
7519    rtx ops[3];
7520    ops[0] = operands[0];
7521    ops[1] = operands[1];
7522    ops[2] = operands[2];
7523    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7524    return "";
7525 }
7526   [(set_attr "length" "4")])
7527
7528 ;;
7529 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7530 ;;
7531 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7532   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7533         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7534                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7535          VLDRBGOQ))
7536   ]
7537   "TARGET_HAVE_MVE"
7538 {
7539    rtx ops[3];
7540    ops[0] = operands[0];
7541    ops[1] = operands[1];
7542    ops[2] = operands[2];
7543    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7544      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7545    else
7546      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7547    return "";
7548 }
7549   [(set_attr "length" "4")])
7550
7551 ;;
7552 ;; [vldrbq_s vldrbq_u]
7553 ;;
7554 (define_insn "mve_vldrbq_<supf><mode>"
7555   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7556         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7557          VLDRBQ))
7558   ]
7559   "TARGET_HAVE_MVE"
7560 {
7561    rtx ops[2];
7562    int regno = REGNO (operands[0]);
7563    ops[0] = gen_rtx_REG (TImode, regno);
7564    ops[1]  = operands[1];
7565    if (<V_sz_elem> == 8)
7566      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7567    else
7568      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7569    return "";
7570 }
7571   [(set_attr "length" "4")])
7572
7573 ;;
7574 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7575 ;;
7576 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7577   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7578         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7579                       (match_operand:SI 2 "immediate_operand" "i")]
7580          VLDRWGBQ))
7581   ]
7582   "TARGET_HAVE_MVE"
7583 {
7584    rtx ops[3];
7585    ops[0] = operands[0];
7586    ops[1] = operands[1];
7587    ops[2] = operands[2];
7588    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7589    return "";
7590 }
7591   [(set_attr "length" "4")])
7592
7593 ;;
7594 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7595 ;;
7596 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7597   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7598    (match_operand:MVE_2 1 "s_register_operand")
7599    (match_operand:MVE_2 2 "s_register_operand")
7600    (match_operand:HI 3 "vpr_register_operand" "Up")
7601    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7602   "TARGET_HAVE_MVE"
7603 {
7604   rtx ind = XEXP (operands[0], 0);
7605   gcc_assert (REG_P (ind));
7606   emit_insn (
7607     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7608                                                        operands[2],
7609                                                        operands[3]));
7610   DONE;
7611 })
7612
7613 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7614   [(set (mem:BLK (scratch))
7615         (unspec:BLK
7616           [(match_operand:SI 0 "register_operand" "r")
7617            (match_operand:MVE_2 1 "s_register_operand" "w")
7618            (match_operand:MVE_2 2 "s_register_operand" "w")
7619            (match_operand:HI 3 "vpr_register_operand" "Up")]
7620           VSTRBSOQ))]
7621   "TARGET_HAVE_MVE"
7622   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7623   [(set_attr "length" "8")])
7624
7625 ;;
7626 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7627 ;;
7628 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7629   [(set (mem:BLK (scratch))
7630         (unspec:BLK
7631                 [(match_operand:V4SI 0 "s_register_operand" "w")
7632                  (match_operand:SI 1 "immediate_operand" "i")
7633                  (match_operand:V4SI 2 "s_register_operand" "w")
7634                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7635          VSTRWSBQ))
7636   ]
7637   "TARGET_HAVE_MVE"
7638 {
7639    rtx ops[3];
7640    ops[0] = operands[0];
7641    ops[1] = operands[1];
7642    ops[2] = operands[2];
7643    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7644    return "";
7645 }
7646   [(set_attr "length" "8")])
7647
7648 ;;
7649 ;; [vstrbq_p_s vstrbq_p_u]
7650 ;;
7651 (define_insn "mve_vstrbq_p_<supf><mode>"
7652   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7653         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7654                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7655          VSTRBQ))
7656   ]
7657   "TARGET_HAVE_MVE"
7658 {
7659    rtx ops[2];
7660    int regno = REGNO (operands[1]);
7661    ops[1] = gen_rtx_REG (TImode, regno);
7662    ops[0]  = operands[0];
7663    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7664    return "";
7665 }
7666   [(set_attr "length" "8")])
7667
7668 ;;
7669 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7670 ;;
7671 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7672   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7673         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7674                        (match_operand:MVE_2 2 "s_register_operand" "w")
7675                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7676          VLDRBGOQ))
7677   ]
7678   "TARGET_HAVE_MVE"
7679 {
7680    rtx ops[4];
7681    ops[0] = operands[0];
7682    ops[1] = operands[1];
7683    ops[2] = operands[2];
7684    ops[3] = operands[3];
7685    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7686      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7687    else
7688      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7689    return "";
7690 }
7691   [(set_attr "length" "8")])
7692
7693 ;;
7694 ;; [vldrbq_z_s vldrbq_z_u]
7695 ;;
7696 (define_insn "mve_vldrbq_z_<supf><mode>"
7697   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7698         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7699                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7700          VLDRBQ))
7701   ]
7702   "TARGET_HAVE_MVE"
7703 {
7704    rtx ops[2];
7705    int regno = REGNO (operands[0]);
7706    ops[0] = gen_rtx_REG (TImode, regno);
7707    ops[1]  = operands[1];
7708    if (<V_sz_elem> == 8)
7709      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7710    else
7711      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7712    return "";
7713 }
7714   [(set_attr "length" "8")])
7715
7716 ;;
7717 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7718 ;;
7719 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7720   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7721         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7722                       (match_operand:SI 2 "immediate_operand" "i")
7723                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7724          VLDRWGBQ))
7725   ]
7726   "TARGET_HAVE_MVE"
7727 {
7728    rtx ops[3];
7729    ops[0] = operands[0];
7730    ops[1] = operands[1];
7731    ops[2] = operands[2];
7732    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7733    return "";
7734 }
7735   [(set_attr "length" "8")])
7736
7737 ;;
7738 ;; [vldrhq_f]
7739 ;;
7740 (define_insn "mve_vldrhq_fv8hf"
7741   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7742         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7743          VLDRHQ_F))
7744   ]
7745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7746 {
7747    rtx ops[2];
7748    int regno = REGNO (operands[0]);
7749    ops[0] = gen_rtx_REG (TImode, regno);
7750    ops[1]  = operands[1];
7751    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7752    return "";
7753 }
7754   [(set_attr "length" "4")])
7755
7756 ;;
7757 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7758 ;;
7759 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7760   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7761         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7762                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7763         VLDRHGOQ))
7764   ]
7765   "TARGET_HAVE_MVE"
7766 {
7767    rtx ops[3];
7768    ops[0] = operands[0];
7769    ops[1] = operands[1];
7770    ops[2] = operands[2];
7771    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7772      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7773    else
7774      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7775    return "";
7776 }
7777   [(set_attr "length" "4")])
7778
7779 ;;
7780 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7781 ;;
7782 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7783   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7784         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7785                        (match_operand:MVE_6 2 "s_register_operand" "w")
7786                        (match_operand:HI 3 "vpr_register_operand" "Up")
7787         ]VLDRHGOQ))
7788   ]
7789   "TARGET_HAVE_MVE"
7790 {
7791    rtx ops[4];
7792    ops[0] = operands[0];
7793    ops[1] = operands[1];
7794    ops[2] = operands[2];
7795    ops[3] = operands[3];
7796    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7797      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7798    else
7799      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7800    return "";
7801 }
7802  [(set_attr "length" "8")])
7803
7804 ;;
7805 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7806 ;;
7807 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7808   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7809         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7810                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7811         VLDRHGSOQ))
7812   ]
7813   "TARGET_HAVE_MVE"
7814 {
7815    rtx ops[3];
7816    ops[0] = operands[0];
7817    ops[1] = operands[1];
7818    ops[2] = operands[2];
7819       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7820      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7821    else
7822      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7823    return "";
7824 }
7825   [(set_attr "length" "4")])
7826
7827 ;;
7828 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7829 ;;
7830 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7831   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7832         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7833                        (match_operand:MVE_6 2 "s_register_operand" "w")
7834                        (match_operand:HI 3 "vpr_register_operand" "Up")
7835         ]VLDRHGSOQ))
7836   ]
7837   "TARGET_HAVE_MVE"
7838 {
7839    rtx ops[4];
7840    ops[0] = operands[0];
7841    ops[1] = operands[1];
7842    ops[2] = operands[2];
7843    ops[3] = operands[3];
7844    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7845      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7846    else
7847      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7848    return "";
7849 }
7850   [(set_attr "length" "8")])
7851
7852 ;;
7853 ;; [vldrhq_s, vldrhq_u]
7854 ;;
7855 (define_insn "mve_vldrhq_<supf><mode>"
7856   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7857         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7858          VLDRHQ))
7859   ]
7860   "TARGET_HAVE_MVE"
7861 {
7862    rtx ops[2];
7863    int regno = REGNO (operands[0]);
7864    ops[0] = gen_rtx_REG (TImode, regno);
7865    ops[1]  = operands[1];
7866    if (<V_sz_elem> == 16)
7867      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7868    else
7869      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7870    return "";
7871 }
7872   [(set_attr "length" "4")])
7873
7874 ;;
7875 ;; [vldrhq_z_f]
7876 ;;
7877 (define_insn "mve_vldrhq_z_fv8hf"
7878   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7879         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7880         (match_operand:HI 2 "vpr_register_operand" "Up")]
7881          VLDRHQ_F))
7882   ]
7883   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7884 {
7885    rtx ops[2];
7886    int regno = REGNO (operands[0]);
7887    ops[0] = gen_rtx_REG (TImode, regno);
7888    ops[1]  = operands[1];
7889    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7890    return "";
7891 }
7892   [(set_attr "length" "8")])
7893
7894 ;;
7895 ;; [vldrhq_z_s vldrhq_z_u]
7896 ;;
7897 (define_insn "mve_vldrhq_z_<supf><mode>"
7898   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7899         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7900         (match_operand:HI 2 "vpr_register_operand" "Up")]
7901          VLDRHQ))
7902   ]
7903   "TARGET_HAVE_MVE"
7904 {
7905    rtx ops[2];
7906    int regno = REGNO (operands[0]);
7907    ops[0] = gen_rtx_REG (TImode, regno);
7908    ops[1]  = operands[1];
7909    if (<V_sz_elem> == 16)
7910      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7911    else
7912      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7913    return "";
7914 }
7915   [(set_attr "length" "8")])
7916
7917 ;;
7918 ;; [vldrwq_f]
7919 ;;
7920 (define_insn "mve_vldrwq_fv4sf"
7921   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7922         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7923          VLDRWQ_F))
7924   ]
7925   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7926 {
7927    rtx ops[2];
7928    int regno = REGNO (operands[0]);
7929    ops[0] = gen_rtx_REG (TImode, regno);
7930    ops[1]  = operands[1];
7931    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7932    return "";
7933 }
7934   [(set_attr "length" "4")])
7935
7936 ;;
7937 ;; [vldrwq_s vldrwq_u]
7938 ;;
7939 (define_insn "mve_vldrwq_<supf>v4si"
7940   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7941         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
7942          VLDRWQ))
7943   ]
7944   "TARGET_HAVE_MVE"
7945 {
7946    rtx ops[2];
7947    int regno = REGNO (operands[0]);
7948    ops[0] = gen_rtx_REG (TImode, regno);
7949    ops[1]  = operands[1];
7950    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7951    return "";
7952 }
7953   [(set_attr "length" "4")])
7954
7955 ;;
7956 ;; [vldrwq_z_f]
7957 ;;
7958 (define_insn "mve_vldrwq_z_fv4sf"
7959   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7960         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
7961         (match_operand:HI 2 "vpr_register_operand" "Up")]
7962          VLDRWQ_F))
7963   ]
7964   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7965 {
7966    rtx ops[2];
7967    int regno = REGNO (operands[0]);
7968    ops[0] = gen_rtx_REG (TImode, regno);
7969    ops[1]  = operands[1];
7970    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7971    return "";
7972 }
7973   [(set_attr "length" "8")])
7974
7975 ;;
7976 ;; [vldrwq_z_s vldrwq_z_u]
7977 ;;
7978 (define_insn "mve_vldrwq_z_<supf>v4si"
7979   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7980         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
7981         (match_operand:HI 2 "vpr_register_operand" "Up")]
7982          VLDRWQ))
7983   ]
7984   "TARGET_HAVE_MVE"
7985 {
7986    rtx ops[2];
7987    int regno = REGNO (operands[0]);
7988    ops[0] = gen_rtx_REG (TImode, regno);
7989    ops[1]  = operands[1];
7990    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7991    return "";
7992 }
7993   [(set_attr "length" "8")])
7994
7995 (define_expand "mve_vld1q_f<mode>"
7996   [(match_operand:MVE_0 0 "s_register_operand")
7997    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7998   ]
7999   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8000 {
8001   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8002   DONE;
8003 })
8004
8005 (define_expand "mve_vld1q_<supf><mode>"
8006   [(match_operand:MVE_2 0 "s_register_operand")
8007    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
8008   ]
8009   "TARGET_HAVE_MVE"
8010 {
8011   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8012   DONE;
8013 })
8014
8015 ;;
8016 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8017 ;;
8018 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8019   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8020         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8021                       (match_operand:SI 2 "immediate_operand" "i")]
8022          VLDRDGBQ))
8023   ]
8024   "TARGET_HAVE_MVE"
8025 {
8026    rtx ops[3];
8027    ops[0] = operands[0];
8028    ops[1] = operands[1];
8029    ops[2] = operands[2];
8030    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8031    return "";
8032 }
8033   [(set_attr "length" "4")])
8034
8035 ;;
8036 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8037 ;;
8038 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8039   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8040         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8041                       (match_operand:SI 2 "immediate_operand" "i")
8042                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8043          VLDRDGBQ))
8044   ]
8045   "TARGET_HAVE_MVE"
8046 {
8047    rtx ops[3];
8048    ops[0] = operands[0];
8049    ops[1] = operands[1];
8050    ops[2] = operands[2];
8051    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8052    return "";
8053 }
8054   [(set_attr "length" "8")])
8055
8056 ;;
8057 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8058 ;;
8059 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8060  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8061        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8062                      (match_operand:V2DI 2 "s_register_operand" "w")]
8063         VLDRDGOQ))
8064  ]
8065  "TARGET_HAVE_MVE"
8066 {
8067   rtx ops[3];
8068   ops[0] = operands[0];
8069   ops[1] = operands[1];
8070   ops[2] = operands[2];
8071   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8072   return "";
8073 }
8074  [(set_attr "length" "4")])
8075
8076 ;;
8077 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8078 ;;
8079 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8080  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8081        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8082                      (match_operand:V2DI 2 "s_register_operand" "w")
8083                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8084         VLDRDGOQ))
8085  ]
8086  "TARGET_HAVE_MVE"
8087 {
8088   rtx ops[3];
8089   ops[0] = operands[0];
8090   ops[1] = operands[1];
8091   ops[2] = operands[2];
8092   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8093   return "";
8094 }
8095  [(set_attr "length" "8")])
8096
8097 ;;
8098 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8099 ;;
8100 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8101   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8102         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8103                       (match_operand:V2DI 2 "s_register_operand" "w")]
8104          VLDRDGSOQ))
8105   ]
8106   "TARGET_HAVE_MVE"
8107 {
8108    rtx ops[3];
8109    ops[0] = operands[0];
8110    ops[1] = operands[1];
8111    ops[2] = operands[2];
8112    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8113    return "";
8114 }
8115   [(set_attr "length" "4")])
8116
8117 ;;
8118 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8119 ;;
8120 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8121   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8122         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8123                       (match_operand:V2DI 2 "s_register_operand" "w")
8124                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8125          VLDRDGSOQ))
8126   ]
8127   "TARGET_HAVE_MVE"
8128 {
8129    rtx ops[3];
8130    ops[0] = operands[0];
8131    ops[1] = operands[1];
8132    ops[2] = operands[2];
8133    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8134    return "";
8135 }
8136   [(set_attr "length" "8")])
8137
8138 ;;
8139 ;; [vldrhq_gather_offset_f]
8140 ;;
8141 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8142   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8143         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8144                       (match_operand:V8HI 2 "s_register_operand" "w")]
8145          VLDRHQGO_F))
8146   ]
8147   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8148 {
8149    rtx ops[3];
8150    ops[0] = operands[0];
8151    ops[1] = operands[1];
8152    ops[2] = operands[2];
8153    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8154    return "";
8155 }
8156   [(set_attr "length" "4")])
8157
8158 ;;
8159 ;; [vldrhq_gather_offset_z_f]
8160 ;;
8161 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8162   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8163         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8164                       (match_operand:V8HI 2 "s_register_operand" "w")
8165                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8166          VLDRHQGO_F))
8167   ]
8168   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8169 {
8170    rtx ops[4];
8171    ops[0] = operands[0];
8172    ops[1] = operands[1];
8173    ops[2] = operands[2];
8174    ops[3] = operands[3];
8175    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8176    return "";
8177 }
8178   [(set_attr "length" "8")])
8179
8180 ;;
8181 ;; [vldrhq_gather_shifted_offset_f]
8182 ;;
8183 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8184   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8185         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8186                       (match_operand:V8HI 2 "s_register_operand" "w")]
8187          VLDRHQGSO_F))
8188   ]
8189   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8190 {
8191    rtx ops[3];
8192    ops[0] = operands[0];
8193    ops[1] = operands[1];
8194    ops[2] = operands[2];
8195    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8196    return "";
8197 }
8198   [(set_attr "length" "4")])
8199
8200 ;;
8201 ;; [vldrhq_gather_shifted_offset_z_f]
8202 ;;
8203 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8204   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8205         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8206                       (match_operand:V8HI 2 "s_register_operand" "w")
8207                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8208          VLDRHQGSO_F))
8209   ]
8210   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8211 {
8212    rtx ops[4];
8213    ops[0] = operands[0];
8214    ops[1] = operands[1];
8215    ops[2] = operands[2];
8216    ops[3] = operands[3];
8217    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8218    return "";
8219 }
8220   [(set_attr "length" "8")])
8221
8222 ;;
8223 ;; [vldrwq_gather_base_f]
8224 ;;
8225 (define_insn "mve_vldrwq_gather_base_fv4sf"
8226   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8227         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8228                       (match_operand:SI 2 "immediate_operand" "i")]
8229          VLDRWQGB_F))
8230   ]
8231   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8232 {
8233    rtx ops[3];
8234    ops[0] = operands[0];
8235    ops[1] = operands[1];
8236    ops[2] = operands[2];
8237    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8238    return "";
8239 }
8240   [(set_attr "length" "4")])
8241
8242 ;;
8243 ;; [vldrwq_gather_base_z_f]
8244 ;;
8245 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8246   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8247         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8248                       (match_operand:SI 2 "immediate_operand" "i")
8249                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8250          VLDRWQGB_F))
8251   ]
8252   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8253 {
8254    rtx ops[3];
8255    ops[0] = operands[0];
8256    ops[1] = operands[1];
8257    ops[2] = operands[2];
8258    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8259    return "";
8260 }
8261   [(set_attr "length" "8")])
8262
8263 ;;
8264 ;; [vldrwq_gather_offset_f]
8265 ;;
8266 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8267   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8268         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8269                        (match_operand:V4SI 2 "s_register_operand" "w")]
8270          VLDRWQGO_F))
8271   ]
8272   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8273 {
8274    rtx ops[3];
8275    ops[0] = operands[0];
8276    ops[1] = operands[1];
8277    ops[2] = operands[2];
8278    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8279    return "";
8280 }
8281   [(set_attr "length" "4")])
8282
8283 ;;
8284 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8285 ;;
8286 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8287   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8288         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8289                        (match_operand:V4SI 2 "s_register_operand" "w")]
8290          VLDRWGOQ))
8291   ]
8292   "TARGET_HAVE_MVE"
8293 {
8294    rtx ops[3];
8295    ops[0] = operands[0];
8296    ops[1] = operands[1];
8297    ops[2] = operands[2];
8298    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8299    return "";
8300 }
8301   [(set_attr "length" "4")])
8302
8303 ;;
8304 ;; [vldrwq_gather_offset_z_f]
8305 ;;
8306 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8307   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8308         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8309                       (match_operand:V4SI 2 "s_register_operand" "w")
8310                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8311          VLDRWQGO_F))
8312   ]
8313   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8314 {
8315    rtx ops[4];
8316    ops[0] = operands[0];
8317    ops[1] = operands[1];
8318    ops[2] = operands[2];
8319    ops[3] = operands[3];
8320    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8321    return "";
8322 }
8323   [(set_attr "length" "8")])
8324
8325 ;;
8326 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8327 ;;
8328 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8329   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8330         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8331                       (match_operand:V4SI 2 "s_register_operand" "w")
8332                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8333          VLDRWGOQ))
8334   ]
8335   "TARGET_HAVE_MVE"
8336 {
8337    rtx ops[4];
8338    ops[0] = operands[0];
8339    ops[1] = operands[1];
8340    ops[2] = operands[2];
8341    ops[3] = operands[3];
8342    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8343    return "";
8344 }
8345   [(set_attr "length" "8")])
8346
8347 ;;
8348 ;; [vldrwq_gather_shifted_offset_f]
8349 ;;
8350 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8351   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8352         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8353                       (match_operand:V4SI 2 "s_register_operand" "w")]
8354          VLDRWQGSO_F))
8355   ]
8356   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8357 {
8358    rtx ops[3];
8359    ops[0] = operands[0];
8360    ops[1] = operands[1];
8361    ops[2] = operands[2];
8362    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8363    return "";
8364 }
8365   [(set_attr "length" "4")])
8366
8367 ;;
8368 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8369 ;;
8370 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8371   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8372         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8373                       (match_operand:V4SI 2 "s_register_operand" "w")]
8374          VLDRWGSOQ))
8375   ]
8376   "TARGET_HAVE_MVE"
8377 {
8378    rtx ops[3];
8379    ops[0] = operands[0];
8380    ops[1] = operands[1];
8381    ops[2] = operands[2];
8382    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8383    return "";
8384 }
8385   [(set_attr "length" "4")])
8386
8387 ;;
8388 ;; [vldrwq_gather_shifted_offset_z_f]
8389 ;;
8390 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8391   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8392         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8393                       (match_operand:V4SI 2 "s_register_operand" "w")
8394                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8395          VLDRWQGSO_F))
8396   ]
8397   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8398 {
8399    rtx ops[4];
8400    ops[0] = operands[0];
8401    ops[1] = operands[1];
8402    ops[2] = operands[2];
8403    ops[3] = operands[3];
8404    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8405    return "";
8406 }
8407   [(set_attr "length" "8")])
8408
8409 ;;
8410 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8411 ;;
8412 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8413   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8414         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8415                       (match_operand:V4SI 2 "s_register_operand" "w")
8416                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8417          VLDRWGSOQ))
8418   ]
8419   "TARGET_HAVE_MVE"
8420 {
8421    rtx ops[4];
8422    ops[0] = operands[0];
8423    ops[1] = operands[1];
8424    ops[2] = operands[2];
8425    ops[3] = operands[3];
8426    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8427    return "";
8428 }
8429   [(set_attr "length" "8")])
8430
8431 ;;
8432 ;; [vstrhq_f]
8433 ;;
8434 (define_insn "mve_vstrhq_fv8hf"
8435   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8436         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8437          VSTRHQ_F))
8438   ]
8439   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8440 {
8441    rtx ops[2];
8442    int regno = REGNO (operands[1]);
8443    ops[1] = gen_rtx_REG (TImode, regno);
8444    ops[0]  = operands[0];
8445    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8446    return "";
8447 }
8448   [(set_attr "length" "4")])
8449
8450 ;;
8451 ;; [vstrhq_p_f]
8452 ;;
8453 (define_insn "mve_vstrhq_p_fv8hf"
8454   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8455         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8456                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8457          VSTRHQ_F))
8458   ]
8459   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8460 {
8461    rtx ops[2];
8462    int regno = REGNO (operands[1]);
8463    ops[1] = gen_rtx_REG (TImode, regno);
8464    ops[0]  = operands[0];
8465    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8466    return "";
8467 }
8468   [(set_attr "length" "8")])
8469
8470 ;;
8471 ;; [vstrhq_p_s vstrhq_p_u]
8472 ;;
8473 (define_insn "mve_vstrhq_p_<supf><mode>"
8474   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8475         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8476                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8477          VSTRHQ))
8478   ]
8479   "TARGET_HAVE_MVE"
8480 {
8481    rtx ops[2];
8482    int regno = REGNO (operands[1]);
8483    ops[1] = gen_rtx_REG (TImode, regno);
8484    ops[0]  = operands[0];
8485    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8486    return "";
8487 }
8488   [(set_attr "length" "8")])
8489
8490 ;;
8491 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8492 ;;
8493 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8494   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8495    (match_operand:MVE_6 1 "s_register_operand")
8496    (match_operand:MVE_6 2 "s_register_operand")
8497    (match_operand:HI 3 "vpr_register_operand")
8498    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8499   "TARGET_HAVE_MVE"
8500 {
8501   rtx ind = XEXP (operands[0], 0);
8502   gcc_assert (REG_P (ind));
8503   emit_insn (
8504     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8505                                                        operands[2],
8506                                                        operands[3]));
8507   DONE;
8508 })
8509
8510 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8511   [(set (mem:BLK (scratch))
8512         (unspec:BLK
8513           [(match_operand:SI 0 "register_operand" "r")
8514            (match_operand:MVE_6 1 "s_register_operand" "w")
8515            (match_operand:MVE_6 2 "s_register_operand" "w")
8516            (match_operand:HI 3 "vpr_register_operand" "Up")]
8517           VSTRHSOQ))]
8518   "TARGET_HAVE_MVE"
8519   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8520   [(set_attr "length" "8")])
8521
8522 ;;
8523 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8524 ;;
8525 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8526   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8527    (match_operand:MVE_6 1 "s_register_operand")
8528    (match_operand:MVE_6 2 "s_register_operand")
8529    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8530   "TARGET_HAVE_MVE"
8531 {
8532   rtx ind = XEXP (operands[0], 0);
8533   gcc_assert (REG_P (ind));
8534   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8535                                                               operands[2]));
8536   DONE;
8537 })
8538
8539 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8540   [(set (mem:BLK (scratch))
8541         (unspec:BLK
8542           [(match_operand:SI 0 "register_operand" "r")
8543            (match_operand:MVE_6 1 "s_register_operand" "w")
8544            (match_operand:MVE_6 2 "s_register_operand" "w")]
8545           VSTRHSOQ))]
8546   "TARGET_HAVE_MVE"
8547   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8548   [(set_attr "length" "4")])
8549
8550 ;;
8551 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8552 ;;
8553 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8554   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8555    (match_operand:MVE_6 1 "s_register_operand")
8556    (match_operand:MVE_6 2 "s_register_operand")
8557    (match_operand:HI 3 "vpr_register_operand")
8558    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8559   "TARGET_HAVE_MVE"
8560 {
8561   rtx ind = XEXP (operands[0], 0);
8562   gcc_assert (REG_P (ind));
8563   emit_insn (
8564     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8565                                                                operands[2],
8566                                                                operands[3]));
8567   DONE;
8568 })
8569
8570 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8571   [(set (mem:BLK (scratch))
8572         (unspec:BLK
8573           [(match_operand:SI 0 "register_operand" "r")
8574            (match_operand:MVE_6 1 "s_register_operand" "w")
8575            (match_operand:MVE_6 2 "s_register_operand" "w")
8576            (match_operand:HI 3 "vpr_register_operand" "Up")]
8577           VSTRHSSOQ))]
8578   "TARGET_HAVE_MVE"
8579   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8580   [(set_attr "length" "8")])
8581
8582 ;;
8583 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8584 ;;
8585 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8586   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8587    (match_operand:MVE_6 1 "s_register_operand")
8588    (match_operand:MVE_6 2 "s_register_operand")
8589    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8590   "TARGET_HAVE_MVE"
8591 {
8592   rtx ind = XEXP (operands[0], 0);
8593   gcc_assert (REG_P (ind));
8594   emit_insn (
8595     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8596                                                              operands[2]));
8597   DONE;
8598 })
8599
8600 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8601   [(set (mem:BLK (scratch))
8602         (unspec:BLK
8603           [(match_operand:SI 0 "register_operand" "r")
8604            (match_operand:MVE_6 1 "s_register_operand" "w")
8605            (match_operand:MVE_6 2 "s_register_operand" "w")]
8606           VSTRHSSOQ))]
8607   "TARGET_HAVE_MVE"
8608   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8609   [(set_attr "length" "4")])
8610
8611 ;;
8612 ;; [vstrhq_s, vstrhq_u]
8613 ;;
8614 (define_insn "mve_vstrhq_<supf><mode>"
8615   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8616         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8617          VSTRHQ))
8618   ]
8619   "TARGET_HAVE_MVE"
8620 {
8621    rtx ops[2];
8622    int regno = REGNO (operands[1]);
8623    ops[1] = gen_rtx_REG (TImode, regno);
8624    ops[0]  = operands[0];
8625    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8626    return "";
8627 }
8628   [(set_attr "length" "4")])
8629
8630 ;;
8631 ;; [vstrwq_f]
8632 ;;
8633 (define_insn "mve_vstrwq_fv4sf"
8634   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8635         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8636          VSTRWQ_F))
8637   ]
8638   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8639 {
8640    rtx ops[2];
8641    int regno = REGNO (operands[1]);
8642    ops[1] = gen_rtx_REG (TImode, regno);
8643    ops[0]  = operands[0];
8644    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8645    return "";
8646 }
8647   [(set_attr "length" "4")])
8648
8649 ;;
8650 ;; [vstrwq_p_f]
8651 ;;
8652 (define_insn "mve_vstrwq_p_fv4sf"
8653   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8654         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8655                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8656          VSTRWQ_F))
8657   ]
8658   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8659 {
8660    rtx ops[2];
8661    int regno = REGNO (operands[1]);
8662    ops[1] = gen_rtx_REG (TImode, regno);
8663    ops[0]  = operands[0];
8664    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8665    return "";
8666 }
8667   [(set_attr "length" "8")])
8668
8669 ;;
8670 ;; [vstrwq_p_s vstrwq_p_u]
8671 ;;
8672 (define_insn "mve_vstrwq_p_<supf>v4si"
8673   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8674         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8675                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8676          VSTRWQ))
8677   ]
8678   "TARGET_HAVE_MVE"
8679 {
8680    rtx ops[2];
8681    int regno = REGNO (operands[1]);
8682    ops[1] = gen_rtx_REG (TImode, regno);
8683    ops[0]  = operands[0];
8684    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8685    return "";
8686 }
8687   [(set_attr "length" "8")])
8688
8689 ;;
8690 ;; [vstrwq_s vstrwq_u]
8691 ;;
8692 (define_insn "mve_vstrwq_<supf>v4si"
8693   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8694         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8695          VSTRWQ))
8696   ]
8697   "TARGET_HAVE_MVE"
8698 {
8699    rtx ops[2];
8700    int regno = REGNO (operands[1]);
8701    ops[1] = gen_rtx_REG (TImode, regno);
8702    ops[0]  = operands[0];
8703    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8704    return "";
8705 }
8706   [(set_attr "length" "4")])
8707
8708 (define_expand "mve_vst1q_f<mode>"
8709   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8710    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8711   ]
8712   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8713 {
8714   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8715   DONE;
8716 })
8717
8718 (define_expand "mve_vst1q_<supf><mode>"
8719   [(match_operand:MVE_2 0 "mve_memory_operand")
8720    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8721   ]
8722   "TARGET_HAVE_MVE"
8723 {
8724   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8725   DONE;
8726 })
8727
8728 ;;
8729 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8730 ;;
8731 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8732   [(set (mem:BLK (scratch))
8733         (unspec:BLK
8734                 [(match_operand:V2DI 0 "s_register_operand" "w")
8735                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8736                  (match_operand:V2DI 2 "s_register_operand" "w")
8737                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8738          VSTRDSBQ))
8739   ]
8740   "TARGET_HAVE_MVE"
8741 {
8742    rtx ops[3];
8743    ops[0] = operands[0];
8744    ops[1] = operands[1];
8745    ops[2] = operands[2];
8746    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8747    return "";
8748 }
8749   [(set_attr "length" "8")])
8750
8751 ;;
8752 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8753 ;;
8754 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8755   [(set (mem:BLK (scratch))
8756         (unspec:BLK
8757                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8758                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8759                  (match_operand:V2DI 2 "s_register_operand" "w")]
8760          VSTRDSBQ))
8761   ]
8762   "TARGET_HAVE_MVE"
8763 {
8764    rtx ops[3];
8765    ops[0] = operands[0];
8766    ops[1] = operands[1];
8767    ops[2] = operands[2];
8768    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8769    return "";
8770 }
8771   [(set_attr "length" "4")])
8772
8773 ;;
8774 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8775 ;;
8776 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8777   [(match_operand:V2DI 0 "mve_scatter_memory")
8778    (match_operand:V2DI 1 "s_register_operand")
8779    (match_operand:V2DI 2 "s_register_operand")
8780    (match_operand:HI 3 "vpr_register_operand")
8781    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8782   "TARGET_HAVE_MVE"
8783 {
8784   rtx ind = XEXP (operands[0], 0);
8785   gcc_assert (REG_P (ind));
8786   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8787                                                               operands[2],
8788                                                               operands[3]));
8789   DONE;
8790 })
8791
8792 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8793   [(set (mem:BLK (scratch))
8794         (unspec:BLK
8795           [(match_operand:SI 0 "register_operand" "r")
8796            (match_operand:V2DI 1 "s_register_operand" "w")
8797            (match_operand:V2DI 2 "s_register_operand" "w")
8798            (match_operand:HI 3 "vpr_register_operand" "Up")]
8799           VSTRDSOQ))]
8800   "TARGET_HAVE_MVE"
8801   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8802   [(set_attr "length" "8")])
8803
8804 ;;
8805 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8806 ;;
8807 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8808   [(match_operand:V2DI 0 "mve_scatter_memory")
8809    (match_operand:V2DI 1 "s_register_operand")
8810    (match_operand:V2DI 2 "s_register_operand")
8811    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8812   "TARGET_HAVE_MVE"
8813 {
8814   rtx ind = XEXP (operands[0], 0);
8815   gcc_assert (REG_P (ind));
8816   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8817                                                             operands[2]));
8818   DONE;
8819 })
8820
8821 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8822   [(set (mem:BLK (scratch))
8823         (unspec:BLK
8824           [(match_operand:SI 0 "register_operand" "r")
8825            (match_operand:V2DI 1 "s_register_operand" "w")
8826            (match_operand:V2DI 2 "s_register_operand" "w")]
8827           VSTRDSOQ))]
8828   "TARGET_HAVE_MVE"
8829   "vstrd.64\t%q2, [%0, %q1]"
8830   [(set_attr "length" "4")])
8831
8832 ;;
8833 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8834 ;;
8835 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8836   [(match_operand:V2DI 0 "mve_scatter_memory")
8837    (match_operand:V2DI 1 "s_register_operand")
8838    (match_operand:V2DI 2 "s_register_operand")
8839    (match_operand:HI 3 "vpr_register_operand")
8840    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8841   "TARGET_HAVE_MVE"
8842 {
8843   rtx ind = XEXP (operands[0], 0);
8844   gcc_assert (REG_P (ind));
8845   emit_insn (
8846     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8847                                                              operands[2],
8848                                                              operands[3]));
8849   DONE;
8850 })
8851
8852 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8853   [(set (mem:BLK (scratch))
8854         (unspec:BLK
8855           [(match_operand:SI 0 "register_operand" "r")
8856            (match_operand:V2DI 1 "s_register_operand" "w")
8857            (match_operand:V2DI 2 "s_register_operand" "w")
8858            (match_operand:HI 3 "vpr_register_operand" "Up")]
8859           VSTRDSSOQ))]
8860   "TARGET_HAVE_MVE"
8861   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8862   [(set_attr "length" "8")])
8863
8864 ;;
8865 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8866 ;;
8867 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8868   [(match_operand:V2DI 0 "mve_scatter_memory")
8869    (match_operand:V2DI 1 "s_register_operand")
8870    (match_operand:V2DI 2 "s_register_operand")
8871    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8872   "TARGET_HAVE_MVE"
8873 {
8874   rtx ind = XEXP (operands[0], 0);
8875   gcc_assert (REG_P (ind));
8876   emit_insn (
8877     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8878                                                            operands[2]));
8879   DONE;
8880 })
8881
8882 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8883   [(set (mem:BLK (scratch))
8884         (unspec:BLK
8885           [(match_operand:SI 0 "register_operand" "r")
8886            (match_operand:V2DI 1 "s_register_operand" "w")
8887            (match_operand:V2DI 2 "s_register_operand" "w")]
8888           VSTRDSSOQ))]
8889   "TARGET_HAVE_MVE"
8890   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8891   [(set_attr "length" "4")])
8892
8893 ;;
8894 ;; [vstrhq_scatter_offset_f]
8895 ;;
8896 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8897   [(match_operand:V8HI 0 "mve_scatter_memory")
8898    (match_operand:V8HI 1 "s_register_operand")
8899    (match_operand:V8HF 2 "s_register_operand")
8900    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8901   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8902 {
8903   rtx ind = XEXP (operands[0], 0);
8904   gcc_assert (REG_P (ind));
8905   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8906                                                        operands[2]));
8907   DONE;
8908 })
8909
8910 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8911   [(set (mem:BLK (scratch))
8912         (unspec:BLK
8913           [(match_operand:SI 0 "register_operand" "r")
8914            (match_operand:V8HI 1 "s_register_operand" "w")
8915            (match_operand:V8HF 2 "s_register_operand" "w")]
8916           VSTRHQSO_F))]
8917   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8918   "vstrh.16\t%q2, [%0, %q1]"
8919   [(set_attr "length" "4")])
8920
8921 ;;
8922 ;; [vstrhq_scatter_offset_p_f]
8923 ;;
8924 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8925   [(match_operand:V8HI 0 "mve_scatter_memory")
8926    (match_operand:V8HI 1 "s_register_operand")
8927    (match_operand:V8HF 2 "s_register_operand")
8928    (match_operand:HI 3 "vpr_register_operand")
8929    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8930   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8931 {
8932   rtx ind = XEXP (operands[0], 0);
8933   gcc_assert (REG_P (ind));
8934   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8935                                                          operands[2],
8936                                                          operands[3]));
8937   DONE;
8938 })
8939
8940 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8941   [(set (mem:BLK (scratch))
8942         (unspec:BLK
8943           [(match_operand:SI 0 "register_operand" "r")
8944            (match_operand:V8HI 1 "s_register_operand" "w")
8945            (match_operand:V8HF 2 "s_register_operand" "w")
8946            (match_operand:HI 3 "vpr_register_operand" "Up")]
8947           VSTRHQSO_F))]
8948   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8949   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8950   [(set_attr "length" "8")])
8951
8952 ;;
8953 ;; [vstrhq_scatter_shifted_offset_f]
8954 ;;
8955 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8956   [(match_operand:V8HI 0 "memory_operand" "=Us")
8957    (match_operand:V8HI 1 "s_register_operand" "w")
8958    (match_operand:V8HF 2 "s_register_operand" "w")
8959    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8960   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8961 {
8962   rtx ind = XEXP (operands[0], 0);
8963   gcc_assert (REG_P (ind));
8964   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8965                                                                operands[2]));
8966   DONE;
8967 })
8968
8969 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8970   [(set (mem:BLK (scratch))
8971         (unspec:BLK
8972           [(match_operand:SI 0 "register_operand" "r")
8973            (match_operand:V8HI 1 "s_register_operand" "w")
8974            (match_operand:V8HF 2 "s_register_operand" "w")]
8975           VSTRHQSSO_F))]
8976   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8977   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8978   [(set_attr "length" "4")])
8979
8980 ;;
8981 ;; [vstrhq_scatter_shifted_offset_p_f]
8982 ;;
8983 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8984   [(match_operand:V8HI 0 "memory_operand" "=Us")
8985    (match_operand:V8HI 1 "s_register_operand" "w")
8986    (match_operand:V8HF 2 "s_register_operand" "w")
8987    (match_operand:HI 3 "vpr_register_operand" "Up")
8988    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8989   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8990 {
8991   rtx ind = XEXP (operands[0], 0);
8992   gcc_assert (REG_P (ind));
8993   emit_insn (
8994     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8995                                                         operands[2],
8996                                                         operands[3]));
8997   DONE;
8998 })
8999
9000 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
9001   [(set (mem:BLK (scratch))
9002         (unspec:BLK
9003           [(match_operand:SI 0 "register_operand" "r")
9004            (match_operand:V8HI 1 "s_register_operand" "w")
9005            (match_operand:V8HF 2 "s_register_operand" "w")
9006            (match_operand:HI 3 "vpr_register_operand" "Up")]
9007           VSTRHQSSO_F))]
9008   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9009   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
9010   [(set_attr "length" "8")])
9011
9012 ;;
9013 ;; [vstrwq_scatter_base_f]
9014 ;;
9015 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9016   [(set (mem:BLK (scratch))
9017         (unspec:BLK
9018                 [(match_operand:V4SI 0 "s_register_operand" "w")
9019                  (match_operand:SI 1 "immediate_operand" "i")
9020                  (match_operand:V4SF 2 "s_register_operand" "w")]
9021          VSTRWQSB_F))
9022   ]
9023   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9024 {
9025    rtx ops[3];
9026    ops[0] = operands[0];
9027    ops[1] = operands[1];
9028    ops[2] = operands[2];
9029    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9030    return "";
9031 }
9032   [(set_attr "length" "4")])
9033
9034 ;;
9035 ;; [vstrwq_scatter_base_p_f]
9036 ;;
9037 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9038   [(set (mem:BLK (scratch))
9039         (unspec:BLK
9040                 [(match_operand:V4SI 0 "s_register_operand" "w")
9041                  (match_operand:SI 1 "immediate_operand" "i")
9042                  (match_operand:V4SF 2 "s_register_operand" "w")
9043                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9044          VSTRWQSB_F))
9045   ]
9046   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9047 {
9048    rtx ops[3];
9049    ops[0] = operands[0];
9050    ops[1] = operands[1];
9051    ops[2] = operands[2];
9052    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9053    return "";
9054 }
9055   [(set_attr "length" "8")])
9056
9057 ;;
9058 ;; [vstrwq_scatter_offset_f]
9059 ;;
9060 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9061   [(match_operand:V4SI 0 "mve_scatter_memory")
9062    (match_operand:V4SI 1 "s_register_operand")
9063    (match_operand:V4SF 2 "s_register_operand")
9064    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9065   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9066 {
9067   rtx ind = XEXP (operands[0], 0);
9068   gcc_assert (REG_P (ind));
9069   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9070                                                        operands[2]));
9071   DONE;
9072 })
9073
9074 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9075   [(set (mem:BLK (scratch))
9076         (unspec:BLK
9077           [(match_operand:SI 0 "register_operand" "r")
9078            (match_operand:V4SI 1 "s_register_operand" "w")
9079            (match_operand:V4SF 2 "s_register_operand" "w")]
9080           VSTRWQSO_F))]
9081   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9082   "vstrw.32\t%q2, [%0, %q1]"
9083   [(set_attr "length" "4")])
9084
9085 ;;
9086 ;; [vstrwq_scatter_offset_p_f]
9087 ;;
9088 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9089   [(match_operand:V4SI 0 "mve_scatter_memory")
9090    (match_operand:V4SI 1 "s_register_operand")
9091    (match_operand:V4SF 2 "s_register_operand")
9092    (match_operand:HI 3 "vpr_register_operand")
9093    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9094   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9095 {
9096   rtx ind = XEXP (operands[0], 0);
9097   gcc_assert (REG_P (ind));
9098   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9099                                                          operands[2],
9100                                                          operands[3]));
9101   DONE;
9102 })
9103
9104 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9105   [(set (mem:BLK (scratch))
9106         (unspec:BLK
9107           [(match_operand:SI 0 "register_operand" "r")
9108            (match_operand:V4SI 1 "s_register_operand" "w")
9109            (match_operand:V4SF 2 "s_register_operand" "w")
9110            (match_operand:HI 3 "vpr_register_operand" "Up")]
9111           VSTRWQSO_F))]
9112   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9113   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9114   [(set_attr "length" "8")])
9115
9116 ;;
9117 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9118 ;;
9119 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9120   [(match_operand:V4SI 0 "mve_scatter_memory")
9121    (match_operand:V4SI 1 "s_register_operand")
9122    (match_operand:V4SI 2 "s_register_operand")
9123    (match_operand:HI 3 "vpr_register_operand")
9124    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9125   "TARGET_HAVE_MVE"
9126 {
9127   rtx ind = XEXP (operands[0], 0);
9128   gcc_assert (REG_P (ind));
9129   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9130                                                               operands[2],
9131                                                               operands[3]));
9132   DONE;
9133 })
9134
9135 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9136   [(set (mem:BLK (scratch))
9137         (unspec:BLK
9138           [(match_operand:SI 0 "register_operand" "r")
9139            (match_operand:V4SI 1 "s_register_operand" "w")
9140            (match_operand:V4SI 2 "s_register_operand" "w")
9141            (match_operand:HI 3 "vpr_register_operand" "Up")]
9142           VSTRWSOQ))]
9143   "TARGET_HAVE_MVE"
9144   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9145   [(set_attr "length" "8")])
9146
9147 ;;
9148 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9149 ;;
9150 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9151   [(match_operand:V4SI 0 "mve_scatter_memory")
9152    (match_operand:V4SI 1 "s_register_operand")
9153    (match_operand:V4SI 2 "s_register_operand")
9154    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9155   "TARGET_HAVE_MVE"
9156 {
9157   rtx ind = XEXP (operands[0], 0);
9158   gcc_assert (REG_P (ind));
9159   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9160                                                             operands[2]));
9161   DONE;
9162 })
9163
9164 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9165   [(set (mem:BLK (scratch))
9166         (unspec:BLK
9167           [(match_operand:SI 0 "register_operand" "r")
9168            (match_operand:V4SI 1 "s_register_operand" "w")
9169            (match_operand:V4SI 2 "s_register_operand" "w")]
9170           VSTRWSOQ))]
9171   "TARGET_HAVE_MVE"
9172   "vstrw.32\t%q2, [%0, %q1]"
9173   [(set_attr "length" "4")])
9174
9175 ;;
9176 ;; [vstrwq_scatter_shifted_offset_f]
9177 ;;
9178 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9179   [(match_operand:V4SI 0 "mve_scatter_memory")
9180    (match_operand:V4SI 1 "s_register_operand")
9181    (match_operand:V4SF 2 "s_register_operand")
9182    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9183   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9184 {
9185   rtx ind = XEXP (operands[0], 0);
9186   gcc_assert (REG_P (ind));
9187   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9188                                                                operands[2]));
9189   DONE;
9190 })
9191
9192 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9193   [(set (mem:BLK (scratch))
9194         (unspec:BLK
9195           [(match_operand:SI 0 "register_operand" "r")
9196            (match_operand:V4SI 1 "s_register_operand" "w")
9197            (match_operand:V4SF 2 "s_register_operand" "w")]
9198          VSTRWQSSO_F))]
9199   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9200   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9201   [(set_attr "length" "8")])
9202
9203 ;;
9204 ;; [vstrwq_scatter_shifted_offset_p_f]
9205 ;;
9206 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9207   [(match_operand:V4SI 0 "mve_scatter_memory")
9208    (match_operand:V4SI 1 "s_register_operand")
9209    (match_operand:V4SF 2 "s_register_operand")
9210    (match_operand:HI 3 "vpr_register_operand")
9211    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9212   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9213 {
9214   rtx ind = XEXP (operands[0], 0);
9215   gcc_assert (REG_P (ind));
9216   emit_insn (
9217     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9218                                                         operands[2],
9219                                                         operands[3]));
9220   DONE;
9221 })
9222
9223 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9224   [(set (mem:BLK (scratch))
9225         (unspec:BLK
9226           [(match_operand:SI 0 "register_operand" "r")
9227            (match_operand:V4SI 1 "s_register_operand" "w")
9228            (match_operand:V4SF 2 "s_register_operand" "w")
9229            (match_operand:HI 3 "vpr_register_operand" "Up")]
9230           VSTRWQSSO_F))]
9231   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9232   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9233   [(set_attr "length" "8")])
9234
9235 ;;
9236 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9237 ;;
9238 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9239   [(match_operand:V4SI 0 "mve_scatter_memory")
9240    (match_operand:V4SI 1 "s_register_operand")
9241    (match_operand:V4SI 2 "s_register_operand")
9242    (match_operand:HI 3 "vpr_register_operand")
9243    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9244   "TARGET_HAVE_MVE"
9245 {
9246   rtx ind = XEXP (operands[0], 0);
9247   gcc_assert (REG_P (ind));
9248   emit_insn (
9249     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9250                                                              operands[2],
9251                                                              operands[3]));
9252   DONE;
9253 })
9254
9255 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9256   [(set (mem:BLK (scratch))
9257         (unspec:BLK
9258           [(match_operand:SI 0 "register_operand" "r")
9259            (match_operand:V4SI 1 "s_register_operand" "w")
9260            (match_operand:V4SI 2 "s_register_operand" "w")
9261            (match_operand:HI 3 "vpr_register_operand" "Up")]
9262           VSTRWSSOQ))]
9263   "TARGET_HAVE_MVE"
9264   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9265   [(set_attr "length" "8")])
9266
9267 ;;
9268 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9269 ;;
9270 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9271   [(match_operand:V4SI 0 "mve_scatter_memory")
9272    (match_operand:V4SI 1 "s_register_operand")
9273    (match_operand:V4SI 2 "s_register_operand")
9274    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9275   "TARGET_HAVE_MVE"
9276 {
9277   rtx ind = XEXP (operands[0], 0);
9278   gcc_assert (REG_P (ind));
9279   emit_insn (
9280     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9281                                                            operands[2]));
9282   DONE;
9283 })
9284
9285 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9286   [(set (mem:BLK (scratch))
9287         (unspec:BLK
9288           [(match_operand:SI 0 "register_operand" "r")
9289            (match_operand:V4SI 1 "s_register_operand" "w")
9290            (match_operand:V4SI 2 "s_register_operand" "w")]
9291           VSTRWSSOQ))]
9292   "TARGET_HAVE_MVE"
9293   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9294   [(set_attr "length" "4")])
9295
9296 ;;
9297 ;; [vaddq_s, vaddq_u])
9298 ;;
9299 (define_insn "mve_vaddq<mode>"
9300   [
9301    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9302         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9303                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9304   ]
9305   "TARGET_HAVE_MVE"
9306   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9307   [(set_attr "type" "mve_move")
9308 ])
9309
9310 ;;
9311 ;; [vaddq_f])
9312 ;;
9313 (define_insn "mve_vaddq_f<mode>"
9314   [
9315    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9316         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9317                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9318   ]
9319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9320   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9321   [(set_attr "type" "mve_move")
9322 ])
9323
9324 ;;
9325 ;; [vidupq_n_u])
9326 ;;
9327 (define_expand "mve_vidupq_n_u<mode>"
9328  [(match_operand:MVE_2 0 "s_register_operand")
9329   (match_operand:SI 1 "s_register_operand")
9330   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9331  "TARGET_HAVE_MVE"
9332 {
9333   rtx temp = gen_reg_rtx (SImode);
9334   emit_move_insn (temp, operands[1]);
9335   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9336   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9337                                           operands[2], inc));
9338   DONE;
9339 })
9340
9341 ;;
9342 ;; [vidupq_u_insn])
9343 ;;
9344 (define_insn "mve_vidupq_u<mode>_insn"
9345  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9346        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9347                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9348          VIDUPQ))
9349   (set (match_operand:SI 1 "s_register_operand" "=Te")
9350        (plus:SI (match_dup 2)
9351                 (match_operand:SI 4 "immediate_operand" "i")))]
9352  "TARGET_HAVE_MVE"
9353  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9354
9355 ;;
9356 ;; [vidupq_m_n_u])
9357 ;;
9358 (define_expand "mve_vidupq_m_n_u<mode>"
9359   [(match_operand:MVE_2 0 "s_register_operand")
9360    (match_operand:MVE_2 1 "s_register_operand")
9361    (match_operand:SI 2 "s_register_operand")
9362    (match_operand:SI 3 "mve_imm_selective_upto_8")
9363    (match_operand:HI 4 "vpr_register_operand")]
9364   "TARGET_HAVE_MVE"
9365 {
9366   rtx temp = gen_reg_rtx (SImode);
9367   emit_move_insn (temp, operands[2]);
9368   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9369   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9370                                              operands[2], operands[3],
9371                                              operands[4], inc));
9372   DONE;
9373 })
9374
9375 ;;
9376 ;; [vidupq_m_wb_u_insn])
9377 ;;
9378 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9379  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9380        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9381                       (match_operand:SI 3 "s_register_operand" "2")
9382                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9383                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9384         VIDUPQ_M))
9385   (set (match_operand:SI 2 "s_register_operand" "=Te")
9386        (plus:SI (match_dup 3)
9387                 (match_operand:SI 6 "immediate_operand" "i")))]
9388  "TARGET_HAVE_MVE"
9389  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9390  [(set_attr "length""8")])
9391
9392 ;;
9393 ;; [vddupq_n_u])
9394 ;;
9395 (define_expand "mve_vddupq_n_u<mode>"
9396  [(match_operand:MVE_2 0 "s_register_operand")
9397   (match_operand:SI 1 "s_register_operand")
9398   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9399  "TARGET_HAVE_MVE"
9400 {
9401   rtx temp = gen_reg_rtx (SImode);
9402   emit_move_insn (temp, operands[1]);
9403   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9404   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9405                                           operands[2], inc));
9406   DONE;
9407 })
9408
9409 ;;
9410 ;; [vddupq_u_insn])
9411 ;;
9412 (define_insn "mve_vddupq_u<mode>_insn"
9413  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9414        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9415                       (match_operand:SI 3 "immediate_operand" "i")]
9416         VDDUPQ))
9417   (set (match_operand:SI 1 "s_register_operand" "=Te")
9418        (minus:SI (match_dup 2)
9419                  (match_operand:SI 4 "immediate_operand" "i")))]
9420  "TARGET_HAVE_MVE"
9421  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9422
9423 ;;
9424 ;; [vddupq_m_n_u])
9425 ;;
9426 (define_expand "mve_vddupq_m_n_u<mode>"
9427   [(match_operand:MVE_2 0 "s_register_operand")
9428    (match_operand:MVE_2 1 "s_register_operand")
9429    (match_operand:SI 2 "s_register_operand")
9430    (match_operand:SI 3 "mve_imm_selective_upto_8")
9431    (match_operand:HI 4 "vpr_register_operand")]
9432   "TARGET_HAVE_MVE"
9433 {
9434   rtx temp = gen_reg_rtx (SImode);
9435   emit_move_insn (temp, operands[2]);
9436   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9437   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9438                                              operands[2], operands[3],
9439                                              operands[4], inc));
9440   DONE;
9441 })
9442
9443 ;;
9444 ;; [vddupq_m_wb_u_insn])
9445 ;;
9446 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9447  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9448        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9449                       (match_operand:SI 3 "s_register_operand" "2")
9450                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9451                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9452         VDDUPQ_M))
9453   (set (match_operand:SI 2 "s_register_operand" "=Te")
9454        (minus:SI (match_dup 3)
9455                  (match_operand:SI 6 "immediate_operand" "i")))]
9456  "TARGET_HAVE_MVE"
9457  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9458  [(set_attr "length""8")])
9459
9460 ;;
9461 ;; [vdwdupq_n_u])
9462 ;;
9463 (define_expand "mve_vdwdupq_n_u<mode>"
9464  [(match_operand:MVE_2 0 "s_register_operand")
9465   (match_operand:SI 1 "s_register_operand")
9466   (match_operand:DI 2 "s_register_operand")
9467   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9468  "TARGET_HAVE_MVE"
9469 {
9470   rtx ignore_wb = gen_reg_rtx (SImode);
9471   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9472                                               operands[1], operands[2],
9473                                               operands[3]));
9474   DONE;
9475 })
9476
9477 ;;
9478 ;; [vdwdupq_wb_u])
9479 ;;
9480 (define_expand "mve_vdwdupq_wb_u<mode>"
9481  [(match_operand:SI 0 "s_register_operand")
9482   (match_operand:SI 1 "s_register_operand")
9483   (match_operand:DI 2 "s_register_operand")
9484   (match_operand:SI 3 "mve_imm_selective_upto_8")
9485   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9486  "TARGET_HAVE_MVE"
9487 {
9488   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9489   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9490                                               operands[1], operands[2],
9491                                               operands[3]));
9492   DONE;
9493 })
9494
9495 ;;
9496 ;; [vdwdupq_wb_u_insn])
9497 ;;
9498 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9499   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9500         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9501                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9502                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9503          VDWDUPQ))
9504    (set (match_operand:SI 1 "s_register_operand" "=Te")
9505         (unspec:SI [(match_dup 2)
9506                     (subreg:SI (match_dup 3) 4)
9507                     (match_dup 4)]
9508          VDWDUPQ))]
9509   "TARGET_HAVE_MVE"
9510   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9511 )
9512
9513 ;;
9514 ;; [vdwdupq_m_n_u])
9515 ;;
9516 (define_expand "mve_vdwdupq_m_n_u<mode>"
9517  [(match_operand:MVE_2 0 "s_register_operand")
9518   (match_operand:MVE_2 1 "s_register_operand")
9519   (match_operand:SI 2 "s_register_operand")
9520   (match_operand:DI 3 "s_register_operand")
9521   (match_operand:SI 4 "mve_imm_selective_upto_8")
9522   (match_operand:HI 5 "vpr_register_operand")]
9523  "TARGET_HAVE_MVE"
9524 {
9525   rtx ignore_wb = gen_reg_rtx (SImode);
9526   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9527                                                 operands[1], operands[2],
9528                                                 operands[3], operands[4],
9529                                                 operands[5]));
9530   DONE;
9531 })
9532
9533 ;;
9534 ;; [vdwdupq_m_wb_u])
9535 ;;
9536 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9537  [(match_operand:SI 0 "s_register_operand")
9538   (match_operand:MVE_2 1 "s_register_operand")
9539   (match_operand:SI 2 "s_register_operand")
9540   (match_operand:DI 3 "s_register_operand")
9541   (match_operand:SI 4 "mve_imm_selective_upto_8")
9542   (match_operand:HI 5 "vpr_register_operand")]
9543  "TARGET_HAVE_MVE"
9544 {
9545   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9546   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9547                                                 operands[1], operands[2],
9548                                                 operands[3], operands[4],
9549                                                 operands[5]));
9550   DONE;
9551 })
9552
9553 ;;
9554 ;; [vdwdupq_m_wb_u_insn])
9555 ;;
9556 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9557   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9558         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9559                        (match_operand:SI 3 "s_register_operand" "1")
9560                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9561                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9562                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9563          VDWDUPQ_M))
9564    (set (match_operand:SI 1 "s_register_operand" "=Te")
9565         (unspec:SI [(match_dup 2)
9566                     (match_dup 3)
9567                     (subreg:SI (match_dup 4) 4)
9568                     (match_dup 5)
9569                     (match_dup 6)]
9570          VDWDUPQ_M))
9571   ]
9572   "TARGET_HAVE_MVE"
9573   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9574   [(set_attr "type" "mve_move")
9575    (set_attr "length""8")])
9576
9577 ;;
9578 ;; [viwdupq_n_u])
9579 ;;
9580 (define_expand "mve_viwdupq_n_u<mode>"
9581  [(match_operand:MVE_2 0 "s_register_operand")
9582   (match_operand:SI 1 "s_register_operand")
9583   (match_operand:DI 2 "s_register_operand")
9584   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9585  "TARGET_HAVE_MVE"
9586 {
9587   rtx ignore_wb = gen_reg_rtx (SImode);
9588   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9589                                               operands[1], operands[2],
9590                                               operands[3]));
9591   DONE;
9592 })
9593
9594 ;;
9595 ;; [viwdupq_wb_u])
9596 ;;
9597 (define_expand "mve_viwdupq_wb_u<mode>"
9598  [(match_operand:SI 0 "s_register_operand")
9599   (match_operand:SI 1 "s_register_operand")
9600   (match_operand:DI 2 "s_register_operand")
9601   (match_operand:SI 3 "mve_imm_selective_upto_8")
9602   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9603  "TARGET_HAVE_MVE"
9604 {
9605   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9606   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9607                                               operands[1], operands[2],
9608                                               operands[3]));
9609   DONE;
9610 })
9611
9612 ;;
9613 ;; [viwdupq_wb_u_insn])
9614 ;;
9615 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9616   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9617         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9618                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9619                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9620          VIWDUPQ))
9621    (set (match_operand:SI 1 "s_register_operand" "=Te")
9622         (unspec:SI [(match_dup 2)
9623                     (subreg:SI (match_dup 3) 4)
9624                     (match_dup 4)]
9625          VIWDUPQ))]
9626   "TARGET_HAVE_MVE"
9627   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9628 )
9629
9630 ;;
9631 ;; [viwdupq_m_n_u])
9632 ;;
9633 (define_expand "mve_viwdupq_m_n_u<mode>"
9634  [(match_operand:MVE_2 0 "s_register_operand")
9635   (match_operand:MVE_2 1 "s_register_operand")
9636   (match_operand:SI 2 "s_register_operand")
9637   (match_operand:DI 3 "s_register_operand")
9638   (match_operand:SI 4 "mve_imm_selective_upto_8")
9639   (match_operand:HI 5 "vpr_register_operand")]
9640  "TARGET_HAVE_MVE"
9641 {
9642   rtx ignore_wb = gen_reg_rtx (SImode);
9643   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9644                                                 operands[1], operands[2],
9645                                                 operands[3], operands[4],
9646                                                 operands[5]));
9647   DONE;
9648 })
9649
9650 ;;
9651 ;; [viwdupq_m_wb_u])
9652 ;;
9653 (define_expand "mve_viwdupq_m_wb_u<mode>"
9654  [(match_operand:SI 0 "s_register_operand")
9655   (match_operand:MVE_2 1 "s_register_operand")
9656   (match_operand:SI 2 "s_register_operand")
9657   (match_operand:DI 3 "s_register_operand")
9658   (match_operand:SI 4 "mve_imm_selective_upto_8")
9659   (match_operand:HI 5 "vpr_register_operand")]
9660  "TARGET_HAVE_MVE"
9661 {
9662   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9663   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9664                                                 operands[1], operands[2],
9665                                                 operands[3], operands[4],
9666                                                 operands[5]));
9667   DONE;
9668 })
9669
9670 ;;
9671 ;; [viwdupq_m_wb_u_insn])
9672 ;;
9673 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9674   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9675         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9676                        (match_operand:SI 3 "s_register_operand" "1")
9677                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9678                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9679                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9680          VIWDUPQ_M))
9681    (set (match_operand:SI 1 "s_register_operand" "=Te")
9682         (unspec:SI [(match_dup 2)
9683                     (match_dup 3)
9684                     (subreg:SI (match_dup 4) 4)
9685                     (match_dup 5)
9686                     (match_dup 6)]
9687          VIWDUPQ_M))
9688   ]
9689   "TARGET_HAVE_MVE"
9690   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9691   [(set_attr "type" "mve_move")
9692    (set_attr "length""8")])
9693
9694 ;;
9695 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9696 ;;
9697 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9698   [(set (mem:BLK (scratch))
9699         (unspec:BLK
9700                 [(match_operand:V4SI 1 "s_register_operand" "0")
9701                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9702                  (match_operand:V4SI 3 "s_register_operand" "w")]
9703          VSTRWSBWBQ))
9704    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9705         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9706          VSTRWSBWBQ))
9707   ]
9708   "TARGET_HAVE_MVE"
9709 {
9710    rtx ops[3];
9711    ops[0] = operands[1];
9712    ops[1] = operands[2];
9713    ops[2] = operands[3];
9714    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9715    return "";
9716 }
9717   [(set_attr "length" "4")])
9718
9719 ;;
9720 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9721 ;;
9722 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9723  [(set (mem:BLK (scratch))
9724        (unspec:BLK
9725                 [(match_operand:V4SI 1 "s_register_operand" "0")
9726                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9727                  (match_operand:V4SI 3 "s_register_operand" "w")
9728                  (match_operand:HI 4 "vpr_register_operand")]
9729         VSTRWSBWBQ))
9730    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9731         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9732          VSTRWSBWBQ))
9733   ]
9734   "TARGET_HAVE_MVE"
9735 {
9736    rtx ops[3];
9737    ops[0] = operands[1];
9738    ops[1] = operands[2];
9739    ops[2] = operands[3];
9740    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9741    return "";
9742 }
9743   [(set_attr "length" "8")])
9744
9745 ;;
9746 ;; [vstrwq_scatter_base_wb_f]
9747 ;;
9748 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9749  [(set (mem:BLK (scratch))
9750        (unspec:BLK
9751                 [(match_operand:V4SI 1 "s_register_operand" "0")
9752                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9753                  (match_operand:V4SF 3 "s_register_operand" "w")]
9754          VSTRWQSBWB_F))
9755    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9756         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9757          VSTRWQSBWB_F))
9758   ]
9759   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9760 {
9761    rtx ops[3];
9762    ops[0] = operands[1];
9763    ops[1] = operands[2];
9764    ops[2] = operands[3];
9765    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9766    return "";
9767 }
9768   [(set_attr "length" "4")])
9769
9770 ;;
9771 ;; [vstrwq_scatter_base_wb_p_f]
9772 ;;
9773 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9774  [(set (mem:BLK (scratch))
9775        (unspec:BLK
9776                 [(match_operand:V4SI 1 "s_register_operand" "0")
9777                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9778                  (match_operand:V4SF 3 "s_register_operand" "w")
9779                  (match_operand:HI 4 "vpr_register_operand")]
9780         VSTRWQSBWB_F))
9781    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9782         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9783          VSTRWQSBWB_F))
9784   ]
9785   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9786 {
9787    rtx ops[3];
9788    ops[0] = operands[1];
9789    ops[1] = operands[2];
9790    ops[2] = operands[3];
9791    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9792    return "";
9793 }
9794   [(set_attr "length" "8")])
9795
9796 ;;
9797 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9798 ;;
9799 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9800   [(set (mem:BLK (scratch))
9801         (unspec:BLK
9802                 [(match_operand:V2DI 1 "s_register_operand" "0")
9803                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9804                  (match_operand:V2DI 3 "s_register_operand" "w")]
9805          VSTRDSBWBQ))
9806    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9807         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9808          VSTRDSBWBQ))
9809   ]
9810   "TARGET_HAVE_MVE"
9811 {
9812    rtx ops[3];
9813    ops[0] = operands[1];
9814    ops[1] = operands[2];
9815    ops[2] = operands[3];
9816    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9817    return "";
9818 }
9819   [(set_attr "length" "4")])
9820
9821 ;;
9822 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9823 ;;
9824 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9825   [(set (mem:BLK (scratch))
9826         (unspec:BLK
9827                 [(match_operand:V2DI 1 "s_register_operand" "0")
9828                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9829                  (match_operand:V2DI 3 "s_register_operand" "w")
9830                  (match_operand:HI 4 "vpr_register_operand")]
9831          VSTRDSBWBQ))
9832    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9833         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9834          VSTRDSBWBQ))
9835   ]
9836   "TARGET_HAVE_MVE"
9837 {
9838    rtx ops[3];
9839    ops[0] = operands[1];
9840    ops[1] = operands[2];
9841    ops[2] = operands[3];
9842    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9843    return "";
9844 }
9845   [(set_attr "length" "8")])
9846
9847 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9848   [(match_operand:V4SI 0 "s_register_operand")
9849    (match_operand:V4SI 1 "s_register_operand")
9850    (match_operand:SI 2 "mve_vldrd_immediate")
9851    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9852   "TARGET_HAVE_MVE"
9853 {
9854   rtx ignore_result = gen_reg_rtx (V4SImode);
9855   emit_insn (
9856   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9857                                                  operands[1], operands[2]));
9858   DONE;
9859 })
9860
9861 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9862   [(match_operand:V4SI 0 "s_register_operand")
9863    (match_operand:V4SI 1 "s_register_operand")
9864    (match_operand:SI 2 "mve_vldrd_immediate")
9865    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9866   "TARGET_HAVE_MVE"
9867 {
9868   rtx ignore_wb = gen_reg_rtx (V4SImode);
9869   emit_insn (
9870   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9871                                                  operands[1], operands[2]));
9872   DONE;
9873 })
9874
9875 ;;
9876 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9877 ;;
9878 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9879   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9880         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9881                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9882                       (mem:BLK (scratch))]
9883          VLDRWGBWBQ))
9884    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9885         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9886          VLDRWGBWBQ))
9887   ]
9888   "TARGET_HAVE_MVE"
9889 {
9890    rtx ops[3];
9891    ops[0] = operands[0];
9892    ops[1] = operands[2];
9893    ops[2] = operands[3];
9894    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9895    return "";
9896 }
9897   [(set_attr "length" "4")])
9898
9899 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9900   [(match_operand:V4SI 0 "s_register_operand")
9901    (match_operand:V4SI 1 "s_register_operand")
9902    (match_operand:SI 2 "mve_vldrd_immediate")
9903    (match_operand:HI 3 "vpr_register_operand")
9904    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9905   "TARGET_HAVE_MVE"
9906 {
9907   rtx ignore_result = gen_reg_rtx (V4SImode);
9908   emit_insn (
9909   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9910                                                    operands[1], operands[2],
9911                                                    operands[3]));
9912   DONE;
9913 })
9914 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9915   [(match_operand:V4SI 0 "s_register_operand")
9916    (match_operand:V4SI 1 "s_register_operand")
9917    (match_operand:SI 2 "mve_vldrd_immediate")
9918    (match_operand:HI 3 "vpr_register_operand")
9919    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9920   "TARGET_HAVE_MVE"
9921 {
9922   rtx ignore_wb = gen_reg_rtx (V4SImode);
9923   emit_insn (
9924   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9925                                                    operands[1], operands[2],
9926                                                    operands[3]));
9927   DONE;
9928 })
9929
9930 ;;
9931 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9932 ;;
9933 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9934   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9935         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9936                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9937                       (match_operand:HI 4 "vpr_register_operand" "Up")
9938                       (mem:BLK (scratch))]
9939          VLDRWGBWBQ))
9940    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9941         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9942          VLDRWGBWBQ))
9943   ]
9944   "TARGET_HAVE_MVE"
9945 {
9946    rtx ops[3];
9947    ops[0] = operands[0];
9948    ops[1] = operands[2];
9949    ops[2] = operands[3];
9950    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9951    return "";
9952 }
9953   [(set_attr "length" "8")])
9954
9955 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9956   [(match_operand:V4SI 0 "s_register_operand")
9957    (match_operand:V4SI 1 "s_register_operand")
9958    (match_operand:SI 2 "mve_vldrd_immediate")
9959    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9960   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9961 {
9962   rtx ignore_result = gen_reg_rtx (V4SFmode);
9963   emit_insn (
9964   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9965                                             operands[1], operands[2]));
9966   DONE;
9967 })
9968
9969 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9970   [(match_operand:V4SF 0 "s_register_operand")
9971    (match_operand:V4SI 1 "s_register_operand")
9972    (match_operand:SI 2 "mve_vldrd_immediate")
9973    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9974   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9975 {
9976   rtx ignore_wb = gen_reg_rtx (V4SImode);
9977   emit_insn (
9978   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9979                                             operands[1], operands[2]));
9980   DONE;
9981 })
9982
9983 ;;
9984 ;; [vldrwq_gather_base_wb_f]
9985 ;;
9986 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9987   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9988         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9989                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9990                       (mem:BLK (scratch))]
9991          VLDRWQGBWB_F))
9992    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9993         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9994          VLDRWQGBWB_F))
9995   ]
9996   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9997 {
9998    rtx ops[3];
9999    ops[0] = operands[0];
10000    ops[1] = operands[2];
10001    ops[2] = operands[3];
10002    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10003    return "";
10004 }
10005   [(set_attr "length" "4")])
10006
10007 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10008   [(match_operand:V4SI 0 "s_register_operand")
10009    (match_operand:V4SI 1 "s_register_operand")
10010    (match_operand:SI 2 "mve_vldrd_immediate")
10011    (match_operand:HI 3 "vpr_register_operand")
10012    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10013   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10014 {
10015   rtx ignore_result = gen_reg_rtx (V4SFmode);
10016   emit_insn (
10017   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10018                                               operands[1], operands[2],
10019                                               operands[3]));
10020   DONE;
10021 })
10022
10023 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10024   [(match_operand:V4SF 0 "s_register_operand")
10025    (match_operand:V4SI 1 "s_register_operand")
10026    (match_operand:SI 2 "mve_vldrd_immediate")
10027    (match_operand:HI 3 "vpr_register_operand")
10028    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10029   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10030 {
10031   rtx ignore_wb = gen_reg_rtx (V4SImode);
10032   emit_insn (
10033   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10034                                               operands[1], operands[2],
10035                                               operands[3]));
10036   DONE;
10037 })
10038
10039 ;;
10040 ;; [vldrwq_gather_base_wb_z_f]
10041 ;;
10042 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10043   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10044         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10045                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10046                       (match_operand:HI 4 "vpr_register_operand" "Up")
10047                       (mem:BLK (scratch))]
10048          VLDRWQGBWB_F))
10049    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10050         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10051          VLDRWQGBWB_F))
10052   ]
10053   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10054 {
10055    rtx ops[3];
10056    ops[0] = operands[0];
10057    ops[1] = operands[2];
10058    ops[2] = operands[3];
10059    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10060    return "";
10061 }
10062   [(set_attr "length" "8")])
10063
10064 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10065   [(match_operand:V2DI 0 "s_register_operand")
10066    (match_operand:V2DI 1 "s_register_operand")
10067    (match_operand:SI 2 "mve_vldrd_immediate")
10068    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10069   "TARGET_HAVE_MVE"
10070 {
10071   rtx ignore_result = gen_reg_rtx (V2DImode);
10072   emit_insn (
10073   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10074                                                  operands[1], operands[2]));
10075   DONE;
10076 })
10077
10078 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10079   [(match_operand:V2DI 0 "s_register_operand")
10080    (match_operand:V2DI 1 "s_register_operand")
10081    (match_operand:SI 2 "mve_vldrd_immediate")
10082    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10083   "TARGET_HAVE_MVE"
10084 {
10085   rtx ignore_wb = gen_reg_rtx (V2DImode);
10086   emit_insn (
10087   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10088                                                  operands[1], operands[2]));
10089   DONE;
10090 })
10091
10092
10093 ;;
10094 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10095 ;;
10096 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10097   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10098         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10099                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10100                       (mem:BLK (scratch))]
10101          VLDRDGBWBQ))
10102    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10103         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10104          VLDRDGBWBQ))
10105   ]
10106   "TARGET_HAVE_MVE"
10107 {
10108    rtx ops[3];
10109    ops[0] = operands[0];
10110    ops[1] = operands[2];
10111    ops[2] = operands[3];
10112    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10113    return "";
10114 }
10115   [(set_attr "length" "4")])
10116
10117 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10118   [(match_operand:V2DI 0 "s_register_operand")
10119    (match_operand:V2DI 1 "s_register_operand")
10120    (match_operand:SI 2 "mve_vldrd_immediate")
10121    (match_operand:HI 3 "vpr_register_operand")
10122    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10123   "TARGET_HAVE_MVE"
10124 {
10125   rtx ignore_result = gen_reg_rtx (V2DImode);
10126   emit_insn (
10127   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10128                                                    operands[1], operands[2],
10129                                                    operands[3]));
10130   DONE;
10131 })
10132
10133 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10134   [(match_operand:V2DI 0 "s_register_operand")
10135    (match_operand:V2DI 1 "s_register_operand")
10136    (match_operand:SI 2 "mve_vldrd_immediate")
10137    (match_operand:HI 3 "vpr_register_operand")
10138    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10139   "TARGET_HAVE_MVE"
10140 {
10141   rtx ignore_wb = gen_reg_rtx (V2DImode);
10142   emit_insn (
10143   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10144                                                    operands[1], operands[2],
10145                                                    operands[3]));
10146   DONE;
10147 })
10148
10149 (define_insn "get_fpscr_nzcvqc"
10150  [(set (match_operand:SI 0 "register_operand" "=r")
10151    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10152  "TARGET_HAVE_MVE"
10153  "vmrs\\t%0, FPSCR_nzcvqc"
10154   [(set_attr "type" "mve_move")])
10155
10156 (define_insn "set_fpscr_nzcvqc"
10157  [(set (reg:SI VFPCC_REGNUM)
10158    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10159     VUNSPEC_SET_FPSCR_NZCVQC))]
10160  "TARGET_HAVE_MVE"
10161  "vmsr\\tFPSCR_nzcvqc, %0"
10162   [(set_attr "type" "mve_move")])
10163
10164 ;;
10165 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10166 ;;
10167 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10168   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10169         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10170                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10171                       (match_operand:HI 4 "vpr_register_operand" "Up")
10172                       (mem:BLK (scratch))]
10173          VLDRDGBWBQ))
10174    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10175         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10176          VLDRDGBWBQ))
10177   ]
10178   "TARGET_HAVE_MVE"
10179 {
10180    rtx ops[3];
10181    ops[0] = operands[0];
10182    ops[1] = operands[2];
10183    ops[2] = operands[3];
10184    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10185    return "";
10186 }
10187   [(set_attr "length" "8")])
10188 ;;
10189 ;; [vadciq_m_s, vadciq_m_u])
10190 ;;
10191 (define_insn "mve_vadciq_m_<supf>v4si"
10192   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10193         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10194                       (match_operand:V4SI 2 "s_register_operand" "w")
10195                       (match_operand:V4SI 3 "s_register_operand" "w")
10196                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10197          VADCIQ_M))
10198    (set (reg:SI VFPCC_REGNUM)
10199         (unspec:SI [(const_int 0)]
10200          VADCIQ_M))
10201   ]
10202   "TARGET_HAVE_MVE"
10203   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10204   [(set_attr "type" "mve_move")
10205    (set_attr "length" "8")])
10206
10207 ;;
10208 ;; [vadciq_u, vadciq_s])
10209 ;;
10210 (define_insn "mve_vadciq_<supf>v4si"
10211   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10212         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10213                       (match_operand:V4SI 2 "s_register_operand" "w")]
10214          VADCIQ))
10215    (set (reg:SI VFPCC_REGNUM)
10216         (unspec:SI [(const_int 0)]
10217          VADCIQ))
10218   ]
10219   "TARGET_HAVE_MVE"
10220   "vadci.i32\t%q0, %q1, %q2"
10221   [(set_attr "type" "mve_move")
10222    (set_attr "length" "4")])
10223
10224 ;;
10225 ;; [vadcq_m_s, vadcq_m_u])
10226 ;;
10227 (define_insn "mve_vadcq_m_<supf>v4si"
10228   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10229         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10230                       (match_operand:V4SI 2 "s_register_operand" "w")
10231                       (match_operand:V4SI 3 "s_register_operand" "w")
10232                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10233          VADCQ_M))
10234    (set (reg:SI VFPCC_REGNUM)
10235         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10236          VADCQ_M))
10237   ]
10238   "TARGET_HAVE_MVE"
10239   "vpst\;vadct.i32\t%q0, %q2, %q3"
10240   [(set_attr "type" "mve_move")
10241    (set_attr "length" "8")])
10242
10243 ;;
10244 ;; [vadcq_u, vadcq_s])
10245 ;;
10246 (define_insn "mve_vadcq_<supf>v4si"
10247   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10248         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10249                        (match_operand:V4SI 2 "s_register_operand" "w")]
10250          VADCQ))
10251    (set (reg:SI VFPCC_REGNUM)
10252         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10253          VADCQ))
10254   ]
10255   "TARGET_HAVE_MVE"
10256   "vadc.i32\t%q0, %q1, %q2"
10257   [(set_attr "type" "mve_move")
10258    (set_attr "length" "4")
10259    (set_attr "conds" "set")])
10260
10261 ;;
10262 ;; [vsbciq_m_u, vsbciq_m_s])
10263 ;;
10264 (define_insn "mve_vsbciq_m_<supf>v4si"
10265   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10266         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10267                       (match_operand:V4SI 2 "s_register_operand" "w")
10268                       (match_operand:V4SI 3 "s_register_operand" "w")
10269                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10270          VSBCIQ_M))
10271    (set (reg:SI VFPCC_REGNUM)
10272         (unspec:SI [(const_int 0)]
10273          VSBCIQ_M))
10274   ]
10275   "TARGET_HAVE_MVE"
10276   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10277   [(set_attr "type" "mve_move")
10278    (set_attr "length" "8")])
10279
10280 ;;
10281 ;; [vsbciq_s, vsbciq_u])
10282 ;;
10283 (define_insn "mve_vsbciq_<supf>v4si"
10284   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10285         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10286                       (match_operand:V4SI 2 "s_register_operand" "w")]
10287          VSBCIQ))
10288    (set (reg:SI VFPCC_REGNUM)
10289         (unspec:SI [(const_int 0)]
10290          VSBCIQ))
10291   ]
10292   "TARGET_HAVE_MVE"
10293   "vsbci.i32\t%q0, %q1, %q2"
10294   [(set_attr "type" "mve_move")
10295    (set_attr "length" "4")])
10296
10297 ;;
10298 ;; [vsbcq_m_u, vsbcq_m_s])
10299 ;;
10300 (define_insn "mve_vsbcq_m_<supf>v4si"
10301   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10302         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10303                       (match_operand:V4SI 2 "s_register_operand" "w")
10304                       (match_operand:V4SI 3 "s_register_operand" "w")
10305                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10306          VSBCQ_M))
10307    (set (reg:SI VFPCC_REGNUM)
10308         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10309          VSBCQ_M))
10310   ]
10311   "TARGET_HAVE_MVE"
10312   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10313   [(set_attr "type" "mve_move")
10314    (set_attr "length" "8")])
10315
10316 ;;
10317 ;; [vsbcq_s, vsbcq_u])
10318 ;;
10319 (define_insn "mve_vsbcq_<supf>v4si"
10320   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10321         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10322                       (match_operand:V4SI 2 "s_register_operand" "w")]
10323          VSBCQ))
10324    (set (reg:SI VFPCC_REGNUM)
10325         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10326          VSBCQ))
10327   ]
10328   "TARGET_HAVE_MVE"
10329   "vsbc.i32\t%q0, %q1, %q2"
10330   [(set_attr "type" "mve_move")
10331    (set_attr "length" "4")])
10332
10333 ;;
10334 ;; [vst2q])
10335 ;;
10336 (define_insn "mve_vst2q<mode>"
10337   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10338         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10339                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10340          VST2Q))
10341   ]
10342   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10343    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10344 {
10345    rtx ops[4];
10346    int regno = REGNO (operands[1]);
10347    ops[0] = gen_rtx_REG (TImode, regno);
10348    ops[1] = gen_rtx_REG (TImode, regno + 4);
10349    rtx reg  = operands[0];
10350    while (reg && !REG_P (reg))
10351     reg = XEXP (reg, 0);
10352    gcc_assert (REG_P (reg));
10353    ops[2] = reg;
10354    ops[3] = operands[0];
10355    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10356                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10357    return "";
10358 }
10359   [(set_attr "length" "8")])
10360
10361 ;;
10362 ;; [vld2q])
10363 ;;
10364 (define_insn "mve_vld2q<mode>"
10365   [(set (match_operand:OI 0 "s_register_operand" "=w")
10366         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10367                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10368          VLD2Q))
10369   ]
10370   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10371    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10372 {
10373    rtx ops[4];
10374    int regno = REGNO (operands[0]);
10375    ops[0] = gen_rtx_REG (TImode, regno);
10376    ops[1] = gen_rtx_REG (TImode, regno + 4);
10377    rtx reg  = operands[1];
10378    while (reg && !REG_P (reg))
10379     reg = XEXP (reg, 0);
10380    gcc_assert (REG_P (reg));
10381    ops[2] = reg;
10382    ops[3] = operands[1];
10383    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10384                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10385    return "";
10386 }
10387   [(set_attr "length" "8")])
10388
10389 ;;
10390 ;; [vld4q])
10391 ;;
10392 (define_insn "mve_vld4q<mode>"
10393   [(set (match_operand:XI 0 "s_register_operand" "=w")
10394         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10395                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10396          VLD4Q))
10397   ]
10398   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10399    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10400 {
10401    rtx ops[6];
10402    int regno = REGNO (operands[0]);
10403    ops[0] = gen_rtx_REG (TImode, regno);
10404    ops[1] = gen_rtx_REG (TImode, regno+4);
10405    ops[2] = gen_rtx_REG (TImode, regno+8);
10406    ops[3] = gen_rtx_REG (TImode, regno + 12);
10407    rtx reg  = operands[1];
10408    while (reg && !REG_P (reg))
10409     reg = XEXP (reg, 0);
10410    gcc_assert (REG_P (reg));
10411    ops[4] = reg;
10412    ops[5] = operands[1];
10413    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10414                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10415                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10416                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10417    return "";
10418 }
10419   [(set_attr "length" "16")])
10420 ;;
10421 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10422 ;;
10423 (define_insn "mve_vec_extract<mode><V_elem_l>"
10424  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10425    (vec_select:<V_elem>
10426     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10427     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10428   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10429    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10430 {
10431   if (BYTES_BIG_ENDIAN)
10432     {
10433       int elt = INTVAL (operands[2]);
10434       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10435       operands[2] = GEN_INT (elt);
10436     }
10437   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10438 }
10439  [(set_attr "type" "mve_move")])
10440
10441 (define_insn "mve_vec_extractv2didi"
10442  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10443    (vec_select:DI
10444     (match_operand:V2DI 1 "s_register_operand" "w")
10445     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10446   "TARGET_HAVE_MVE"
10447 {
10448   int elt = INTVAL (operands[2]);
10449   if (BYTES_BIG_ENDIAN)
10450     elt = 1 - elt;
10451
10452   if (elt == 0)
10453    return "vmov\t%Q0, %R0, %e1";
10454   else
10455    return "vmov\t%Q0, %R0, %f1";
10456 }
10457  [(set_attr "type" "mve_move")])
10458
10459 (define_insn "*mve_vec_extract_sext_internal<mode>"
10460  [(set (match_operand:SI 0 "s_register_operand" "=r")
10461    (sign_extend:SI
10462     (vec_select:<V_elem>
10463      (match_operand:MVE_2 1 "s_register_operand" "w")
10464      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10465   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10466    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10467 {
10468   if (BYTES_BIG_ENDIAN)
10469     {
10470       int elt = INTVAL (operands[2]);
10471       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10472       operands[2] = GEN_INT (elt);
10473     }
10474   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10475 }
10476  [(set_attr "type" "mve_move")])
10477
10478 (define_insn "*mve_vec_extract_zext_internal<mode>"
10479  [(set (match_operand:SI 0 "s_register_operand" "=r")
10480    (zero_extend:SI
10481     (vec_select:<V_elem>
10482      (match_operand:MVE_2 1 "s_register_operand" "w")
10483      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10484   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10485    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10486 {
10487   if (BYTES_BIG_ENDIAN)
10488     {
10489       int elt = INTVAL (operands[2]);
10490       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10491       operands[2] = GEN_INT (elt);
10492     }
10493   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10494 }
10495  [(set_attr "type" "mve_move")])
10496
10497 ;;
10498 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10499 ;;
10500 (define_insn "mve_vec_set<mode>_internal"
10501  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10502        (vec_merge:VQ2
10503         (vec_duplicate:VQ2
10504           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10505         (match_operand:VQ2 3 "s_register_operand" "0")
10506         (match_operand:SI 2 "immediate_operand" "i")))]
10507   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10508    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10509 {
10510   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10511   if (BYTES_BIG_ENDIAN)
10512     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10513   operands[2] = GEN_INT (elt);
10514
10515   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10516 }
10517  [(set_attr "type" "mve_move")])
10518
10519 (define_insn "mve_vec_setv2di_internal"
10520  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10521        (vec_merge:V2DI
10522         (vec_duplicate:V2DI
10523           (match_operand:DI 1 "nonimmediate_operand" "r"))
10524         (match_operand:V2DI 3 "s_register_operand" "0")
10525         (match_operand:SI 2 "immediate_operand" "i")))]
10526  "TARGET_HAVE_MVE"
10527 {
10528   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10529   if (BYTES_BIG_ENDIAN)
10530     elt = 1 - elt;
10531
10532   if (elt == 0)
10533    return "vmov\t%e0, %Q1, %R1";
10534   else
10535    return "vmov\t%f0, %J1, %K1";
10536 }
10537  [(set_attr "type" "mve_move")])
10538
10539 ;;
10540 ;; [uqrshll_di]
10541 ;;
10542 (define_insn "mve_uqrshll_sat<supf>_di"
10543   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10544         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10545                     (match_operand:SI 2 "register_operand" "r")]
10546          UQRSHLLQ))]
10547   "TARGET_HAVE_MVE"
10548   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10549   [(set_attr "predicable" "yes")])
10550
10551 ;;
10552 ;; [sqrshrl_di]
10553 ;;
10554 (define_insn "mve_sqrshrl_sat<supf>_di"
10555   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10556         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10557                     (match_operand:SI 2 "register_operand" "r")]
10558          SQRSHRLQ))]
10559   "TARGET_HAVE_MVE"
10560   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10561   [(set_attr "predicable" "yes")])
10562
10563 ;;
10564 ;; [uqrshl_si]
10565 ;;
10566 (define_insn "mve_uqrshl_si"
10567   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10568         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10569                     (match_operand:SI 2 "register_operand" "r")]
10570          UQRSHL))]
10571   "TARGET_HAVE_MVE"
10572   "uqrshl%?\\t%1, %2"
10573   [(set_attr "predicable" "yes")])
10574
10575 ;;
10576 ;; [sqrshr_si]
10577 ;;
10578 (define_insn "mve_sqrshr_si"
10579   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10580         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10581                     (match_operand:SI 2 "register_operand" "r")]
10582          SQRSHR))]
10583   "TARGET_HAVE_MVE"
10584   "sqrshr%?\\t%1, %2"
10585   [(set_attr "predicable" "yes")])
10586
10587 ;;
10588 ;; [uqshll_di]
10589 ;;
10590 (define_insn "mve_uqshll_di"
10591   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10592         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10593                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10594   "TARGET_HAVE_MVE"
10595   "uqshll%?\\t%Q1, %R1, %2"
10596   [(set_attr "predicable" "yes")])
10597
10598 ;;
10599 ;; [urshrl_di]
10600 ;;
10601 (define_insn "mve_urshrl_di"
10602   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10603         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10604                     (match_operand:SI 2 "immediate_operand" "Pg")]
10605          URSHRL))]
10606   "TARGET_HAVE_MVE"
10607   "urshrl%?\\t%Q1, %R1, %2"
10608   [(set_attr "predicable" "yes")])
10609
10610 ;;
10611 ;; [uqshl_si]
10612 ;;
10613 (define_insn "mve_uqshl_si"
10614   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10615         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10616                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10617   "TARGET_HAVE_MVE"
10618   "uqshl%?\\t%1, %2"
10619   [(set_attr "predicable" "yes")])
10620
10621 ;;
10622 ;; [urshr_si]
10623 ;;
10624 (define_insn "mve_urshr_si"
10625   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10626         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10627                     (match_operand:SI 2 "immediate_operand" "Pg")]
10628          URSHR))]
10629   "TARGET_HAVE_MVE"
10630   "urshr%?\\t%1, %2"
10631   [(set_attr "predicable" "yes")])
10632
10633 ;;
10634 ;; [sqshl_si]
10635 ;;
10636 (define_insn "mve_sqshl_si"
10637   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10638         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10639                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10640   "TARGET_HAVE_MVE"
10641   "sqshl%?\\t%1, %2"
10642   [(set_attr "predicable" "yes")])
10643
10644 ;;
10645 ;; [srshr_si]
10646 ;;
10647 (define_insn "mve_srshr_si"
10648   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10649         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10650                     (match_operand:SI 2 "immediate_operand" "Pg")]
10651          SRSHR))]
10652   "TARGET_HAVE_MVE"
10653   "srshr%?\\t%1, %2"
10654   [(set_attr "predicable" "yes")])
10655
10656 ;;
10657 ;; [srshrl_di]
10658 ;;
10659 (define_insn "mve_srshrl_di"
10660   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10661         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10662                     (match_operand:SI 2 "immediate_operand" "Pg")]
10663          SRSHRL))]
10664   "TARGET_HAVE_MVE"
10665   "srshrl%?\\t%Q1, %R1, %2"
10666   [(set_attr "predicable" "yes")])
10667
10668 ;;
10669 ;; [sqshll_di]
10670 ;;
10671 (define_insn "mve_sqshll_di"
10672   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10673         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10674                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10675   "TARGET_HAVE_MVE"
10676   "sqshll%?\\t%Q1, %R1, %2"
10677   [(set_attr "predicable" "yes")])
10678
10679 ;;
10680 ;; [vshlcq_m_u vshlcq_m_s]
10681 ;;
10682 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10683  [(match_operand:MVE_2 0 "s_register_operand")
10684   (match_operand:MVE_2 1 "s_register_operand")
10685   (match_operand:SI 2 "s_register_operand")
10686   (match_operand:SI 3 "mve_imm_32")
10687   (match_operand:HI 4 "vpr_register_operand")
10688   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10689  "TARGET_HAVE_MVE"
10690 {
10691   rtx ignore_wb = gen_reg_rtx (SImode);
10692   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10693                                             operands[2], operands[3],
10694                                             operands[4]));
10695   DONE;
10696 })
10697
10698 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10699  [(match_operand:SI 0 "s_register_operand")
10700   (match_operand:MVE_2 1 "s_register_operand")
10701   (match_operand:SI 2 "s_register_operand")
10702   (match_operand:SI 3 "mve_imm_32")
10703   (match_operand:HI 4 "vpr_register_operand")
10704   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10705  "TARGET_HAVE_MVE"
10706 {
10707   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10708   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10709                                             operands[1], operands[2],
10710                                             operands[3], operands[4]));
10711   DONE;
10712 })
10713
10714 (define_insn "mve_vshlcq_m_<supf><mode>"
10715  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10716        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10717                       (match_operand:SI 3 "s_register_operand" "1")
10718                       (match_operand:SI 4 "mve_imm_32" "Rf")
10719                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10720         VSHLCQ_M))
10721   (set (match_operand:SI  1 "s_register_operand" "=r")
10722        (unspec:SI [(match_dup 2)
10723                    (match_dup 3)
10724                    (match_dup 4)
10725                    (match_dup 5)]
10726         VSHLCQ_M))
10727  ]
10728  "TARGET_HAVE_MVE"
10729  "vpst\;vshlct\t%q0, %1, %4"
10730  [(set_attr "type" "mve_move")
10731   (set_attr "length" "8")])
10732
10733 (define_insn "*mve_vec_duplicate<mode>"
10734  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
10735        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
10736  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
10737  "vdup.<V_sz_elem>\t%q0, %1"
10738  [(set_attr "type" "mve_move")])
10739
10740 ;; CDE instructions on MVE registers.
10741
10742 (define_insn "arm_vcx1qv16qi"
10743   [(set (match_operand:V16QI 0 "register_operand" "=t")
10744         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10745                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10746          UNSPEC_VCDE))]
10747   "TARGET_CDE && TARGET_HAVE_MVE"
10748   "vcx1\\tp%c1, %q0, #%c2"
10749   [(set_attr "type" "coproc")]
10750 )
10751
10752 (define_insn "arm_vcx1qav16qi"
10753   [(set (match_operand:V16QI 0 "register_operand" "=t")
10754         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10755                             (match_operand:V16QI 2 "register_operand" "0")
10756                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10757          UNSPEC_VCDEA))]
10758   "TARGET_CDE && TARGET_HAVE_MVE"
10759   "vcx1a\\tp%c1, %q0, #%c3"
10760   [(set_attr "type" "coproc")]
10761 )
10762
10763 (define_insn "arm_vcx2qv16qi"
10764   [(set (match_operand:V16QI 0 "register_operand" "=t")
10765         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10766                           (match_operand:V16QI 2 "register_operand" "t")
10767                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10768          UNSPEC_VCDE))]
10769   "TARGET_CDE && TARGET_HAVE_MVE"
10770   "vcx2\\tp%c1, %q0, %q2, #%c3"
10771   [(set_attr "type" "coproc")]
10772 )
10773
10774 (define_insn "arm_vcx2qav16qi"
10775   [(set (match_operand:V16QI 0 "register_operand" "=t")
10776         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10777                           (match_operand:V16QI 2 "register_operand" "0")
10778                           (match_operand:V16QI 3 "register_operand" "t")
10779                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10780          UNSPEC_VCDEA))]
10781   "TARGET_CDE && TARGET_HAVE_MVE"
10782   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10783   [(set_attr "type" "coproc")]
10784 )
10785
10786 (define_insn "arm_vcx3qv16qi"
10787   [(set (match_operand:V16QI 0 "register_operand" "=t")
10788         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10789                           (match_operand:V16QI 2 "register_operand" "t")
10790                           (match_operand:V16QI 3 "register_operand" "t")
10791                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10792          UNSPEC_VCDE))]
10793   "TARGET_CDE && TARGET_HAVE_MVE"
10794   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10795   [(set_attr "type" "coproc")]
10796 )
10797
10798 (define_insn "arm_vcx3qav16qi"
10799   [(set (match_operand:V16QI 0 "register_operand" "=t")
10800         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10801                           (match_operand:V16QI 2 "register_operand" "0")
10802                           (match_operand:V16QI 3 "register_operand" "t")
10803                           (match_operand:V16QI 4 "register_operand" "t")
10804                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10805          UNSPEC_VCDEA))]
10806   "TARGET_CDE && TARGET_HAVE_MVE"
10807   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10808   [(set_attr "type" "coproc")]
10809 )
10810
10811 (define_insn "arm_vcx1q<a>_p_v16qi"
10812   [(set (match_operand:V16QI 0 "register_operand" "=t")
10813         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10814                            (match_operand:V16QI 2 "register_operand" "0")
10815                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10816                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10817          CDE_VCX))]
10818   "TARGET_CDE && TARGET_HAVE_MVE"
10819   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10820   [(set_attr "type" "coproc")
10821    (set_attr "length" "8")]
10822 )
10823
10824 (define_insn "arm_vcx2q<a>_p_v16qi"
10825   [(set (match_operand:V16QI 0 "register_operand" "=t")
10826         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10827                           (match_operand:V16QI 2 "register_operand" "0")
10828                           (match_operand:V16QI 3 "register_operand" "t")
10829                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10830                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10831          CDE_VCX))]
10832   "TARGET_CDE && TARGET_HAVE_MVE"
10833   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10834   [(set_attr "type" "coproc")
10835    (set_attr "length" "8")]
10836 )
10837
10838 (define_insn "arm_vcx3q<a>_p_v16qi"
10839   [(set (match_operand:V16QI 0 "register_operand" "=t")
10840         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10841                           (match_operand:V16QI 2 "register_operand" "0")
10842                           (match_operand:V16QI 3 "register_operand" "t")
10843                           (match_operand:V16QI 4 "register_operand" "t")
10844                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10845                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10846          CDE_VCX))]
10847   "TARGET_CDE && TARGET_HAVE_MVE"
10848   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10849   [(set_attr "type" "coproc")
10850    (set_attr "length" "8")]
10851 )
10852
10853 (define_insn "*movmisalign<mode>_mve_store"
10854   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Um")
10855         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10856          UNSPEC_MISALIGNED_ACCESS))]
10857   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10858     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10859    && !BYTES_BIG_ENDIAN && unaligned_access"
10860   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10861   [(set_attr "type" "mve_store")]
10862 )
10863
10864
10865 (define_insn "*movmisalign<mode>_mve_load"
10866   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10867         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Um")]
10868          UNSPEC_MISALIGNED_ACCESS))]
10869   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10870     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10871    && !BYTES_BIG_ENDIAN && unaligned_access"
10872   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10873   [(set_attr "type" "mve_load")]
10874 )