gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Uxi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       rtx ops[2];
  45       int regno = (which_alternative == 7)
  46                   ? REGNO (operands[1]) : REGNO (operands[0]);
  47
  48       ops[0] = operands[0];
  49       ops[1] = operands[1];
  50       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode)
  51         {
  52           if (which_alternative == 7)
  53             {
  54               ops[1] = gen_rtx_REG (DImode, regno);
  55               output_asm_insn ("vstr.64\t%P1, %E0",ops);
  56             }
  57           else
  58             {
  59               ops[0] = gen_rtx_REG (DImode, regno);
  60               output_asm_insn ("vldr.64\t%P0, %E1",ops);
  61             }
  62         }
  63       else if (<MODE>mode == TImode)
  64         {
  65           if (which_alternative == 7)
  66             output_asm_insn ("vstr.64\t%q1, %E0",ops);
  67           else
  68             output_asm_insn ("vldr.64\t%q0, %E1",ops);
  69         }
  70       else
  71         {
  72           if (which_alternative == 7)
  73             {
  74               ops[1] = gen_rtx_REG (TImode, regno);
  75               output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0",ops);
  76             }
  77           else
  78             {
  79               ops[0] = gen_rtx_REG (TImode, regno);
  80               output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1",ops);
  81             }
  82         }
  83       return "";
  84     }
  85   switch (which_alternative)
  86     {
  87     case 0:
  88       return "vmov\t%q0, %q1";
  89     case 1:
  90       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  91     case 2:
  92       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  93     case 5:
  94       return output_move_quad (operands);
  95     case 8:
  96         return output_move_neon (operands);
  97     default:
  98       gcc_unreachable ();
  99       return "";
 100     }
 101 }
 102   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
 103    (set_attr "length" "4,8,8,4,8,8,4,4,4")
 104    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
 105    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
 106
 107 (define_insn "*mve_mov<mode>"
 108   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 109         (vec_duplicate:MVE_types
 110           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 111   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 112 {
 113   if (which_alternative == 0)
 114     return "vdup.<V_sz_elem>\t%q0, %1";
 115   return "vmov.<V_sz_elem>\t%q0, %1";
 116 }
 117   [(set_attr "length" "4,4")
 118    (set_attr "type" "mve_move,mve_move")])
 119
 120 ;;
 121 ;; [vst4q])
 122 ;;
 123 (define_insn "mve_vst4q<mode>"
 124   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 125         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 126                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 127          VST4Q))
 128   ]
 129   "TARGET_HAVE_MVE"
 130 {
 131    rtx ops[6];
 132    int regno = REGNO (operands[1]);
 133    ops[0] = gen_rtx_REG (TImode, regno);
 134    ops[1] = gen_rtx_REG (TImode, regno+4);
 135    ops[2] = gen_rtx_REG (TImode, regno+8);
 136    ops[3] = gen_rtx_REG (TImode, regno+12);
 137    rtx reg  = operands[0];
 138    while (reg && !REG_P (reg))
 139     reg = XEXP (reg, 0);
 140    gcc_assert (REG_P (reg));
 141    ops[4] = reg;
 142    ops[5] = operands[0];
 143    /* Here in first three instructions data is stored to ops[4]'s location but
 144       in the fourth instruction data is stored to operands[0], this is to
 145       support the writeback.  */
 146    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 147                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 148                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 149                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 150    return "";
 151 }
 152   [(set_attr "length" "16")])
 153
 154 ;;
 155 ;; [vrndq_m_f])
 156 ;;
 157 (define_insn "mve_vrndq_m_f<mode>"
 158   [
 159    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 160         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 161                        (match_operand:MVE_0 2 "s_register_operand" "w")
 162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 163          VRNDQ_M_F))
 164   ]
 165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 166   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 167   [(set_attr "type" "mve_move")
 168    (set_attr "length""8")])
 169
 170 ;;
 171 ;; [vrndxq_f])
 172 ;;
 173 (define_insn "mve_vrndxq_f<mode>"
 174   [
 175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 177          VRNDXQ_F))
 178   ]
 179   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 180   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 181   [(set_attr "type" "mve_move")
 182 ])
 183
 184 ;;
 185 ;; [vrndq_f])
 186 ;;
 187 (define_insn "mve_vrndq_f<mode>"
 188   [
 189    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 190         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 191          VRNDQ_F))
 192   ]
 193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 194   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 195   [(set_attr "type" "mve_move")
 196 ])
 197
 198 ;;
 199 ;; [vrndpq_f])
 200 ;;
 201 (define_insn "mve_vrndpq_f<mode>"
 202   [
 203    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 204         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 205          VRNDPQ_F))
 206   ]
 207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 208   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 209   [(set_attr "type" "mve_move")
 210 ])
 211
 212 ;;
 213 ;; [vrndnq_f])
 214 ;;
 215 (define_insn "mve_vrndnq_f<mode>"
 216   [
 217    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 218         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 219          VRNDNQ_F))
 220   ]
 221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 222   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 223   [(set_attr "type" "mve_move")
 224 ])
 225
 226 ;;
 227 ;; [vrndmq_f])
 228 ;;
 229 (define_insn "mve_vrndmq_f<mode>"
 230   [
 231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 232         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 233          VRNDMQ_F))
 234   ]
 235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 236   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 237   [(set_attr "type" "mve_move")
 238 ])
 239
 240 ;;
 241 ;; [vrndaq_f])
 242 ;;
 243 (define_insn "mve_vrndaq_f<mode>"
 244   [
 245    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 246         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 247          VRNDAQ_F))
 248   ]
 249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 250   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 251   [(set_attr "type" "mve_move")
 252 ])
 253
 254 ;;
 255 ;; [vrev64q_f])
 256 ;;
 257 (define_insn "mve_vrev64q_f<mode>"
 258   [
 259    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 260         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 261          VREV64Q_F))
 262   ]
 263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 264   "vrev64.%#<V_sz_elem> %q0, %q1"
 265   [(set_attr "type" "mve_move")
 266 ])
 267
 268 ;;
 269 ;; [vnegq_f])
 270 ;;
 271 (define_insn "mve_vnegq_f<mode>"
 272   [
 273    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 274         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 275          VNEGQ_F))
 276   ]
 277   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 278   "vneg.f%#<V_sz_elem>  %q0, %q1"
 279   [(set_attr "type" "mve_move")
 280 ])
 281
 282 ;;
 283 ;; [vdupq_n_f])
 284 ;;
 285 (define_insn "mve_vdupq_n_f<mode>"
 286   [
 287    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 288         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 289          VDUPQ_N_F))
 290   ]
 291   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 292   "vdup.%#<V_sz_elem>   %q0, %1"
 293   [(set_attr "type" "mve_move")
 294 ])
 295
 296 ;;
 297 ;; [vabsq_f])
 298 ;;
 299 (define_insn "mve_vabsq_f<mode>"
 300   [
 301    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 302         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 303          VABSQ_F))
 304   ]
 305   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 306   "vabs.f%#<V_sz_elem>  %q0, %q1"
 307   [(set_attr "type" "mve_move")
 308 ])
 309
 310 ;;
 311 ;; [vrev32q_f])
 312 ;;
 313 (define_insn "mve_vrev32q_fv8hf"
 314   [
 315    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 316         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 317          VREV32Q_F))
 318   ]
 319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 320   "vrev32.16 %q0, %q1"
 321   [(set_attr "type" "mve_move")
 322 ])
 323 ;;
 324 ;; [vcvttq_f32_f16])
 325 ;;
 326 (define_insn "mve_vcvttq_f32_f16v4sf"
 327   [
 328    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 329         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 330          VCVTTQ_F32_F16))
 331   ]
 332   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 333   "vcvtt.f32.f16 %q0, %q1"
 334   [(set_attr "type" "mve_move")
 335 ])
 336
 337 ;;
 338 ;; [vcvtbq_f32_f16])
 339 ;;
 340 (define_insn "mve_vcvtbq_f32_f16v4sf"
 341   [
 342    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 343         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 344          VCVTBQ_F32_F16))
 345   ]
 346   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 347   "vcvtb.f32.f16 %q0, %q1"
 348   [(set_attr "type" "mve_move")
 349 ])
 350
 351 ;;
 352 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 353 ;;
 354 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 355   [
 356    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 357         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 358          VCVTQ_TO_F))
 359   ]
 360   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 361   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 362   [(set_attr "type" "mve_move")
 363 ])
 364
 365 ;;
 366 ;; [vrev64q_u, vrev64q_s])
 367 ;;
 368 (define_insn "mve_vrev64q_<supf><mode>"
 369   [
 370    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 371         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 372          VREV64Q))
 373   ]
 374   "TARGET_HAVE_MVE"
 375   "vrev64.%#<V_sz_elem> %q0, %q1"
 376   [(set_attr "type" "mve_move")
 377 ])
 378
 379 ;;
 380 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 381 ;;
 382 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 383   [
 384    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 385         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 386          VCVTQ_FROM_F))
 387   ]
 388   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 389   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 390   [(set_attr "type" "mve_move")
 391 ])
 392 ;; [vqnegq_s])
 393 ;;
 394 (define_insn "mve_vqnegq_s<mode>"
 395   [
 396    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 397         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 398          VQNEGQ_S))
 399   ]
 400   "TARGET_HAVE_MVE"
 401   "vqneg.s%#<V_sz_elem> %q0, %q1"
 402   [(set_attr "type" "mve_move")
 403 ])
 404
 405 ;;
 406 ;; [vqabsq_s])
 407 ;;
 408 (define_insn "mve_vqabsq_s<mode>"
 409   [
 410    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 411         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 412          VQABSQ_S))
 413   ]
 414   "TARGET_HAVE_MVE"
 415   "vqabs.s%#<V_sz_elem> %q0, %q1"
 416   [(set_attr "type" "mve_move")
 417 ])
 418
 419 ;;
 420 ;; [vnegq_s])
 421 ;;
 422 (define_insn "mve_vnegq_s<mode>"
 423   [
 424    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 425         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 426          VNEGQ_S))
 427   ]
 428   "TARGET_HAVE_MVE"
 429   "vneg.s%#<V_sz_elem>  %q0, %q1"
 430   [(set_attr "type" "mve_move")
 431 ])
 432
 433 ;;
 434 ;; [vmvnq_u, vmvnq_s])
 435 ;;
 436 (define_insn "mve_vmvnq_u<mode>"
 437   [
 438    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 439         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 440   ]
 441   "TARGET_HAVE_MVE"
 442   "vmvn\t%q0, %q1"
 443   [(set_attr "type" "mve_move")
 444 ])
 445 (define_expand "mve_vmvnq_s<mode>"
 446   [
 447    (set (match_operand:MVE_2 0 "s_register_operand")
 448         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 449   ]
 450   "TARGET_HAVE_MVE"
 451 )
 452
 453 ;;
 454 ;; [vdupq_n_u, vdupq_n_s])
 455 ;;
 456 (define_insn "mve_vdupq_n_<supf><mode>"
 457   [
 458    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 459         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 460          VDUPQ_N))
 461   ]
 462   "TARGET_HAVE_MVE"
 463   "vdup.%#<V_sz_elem>   %q0, %1"
 464   [(set_attr "type" "mve_move")
 465 ])
 466
 467 ;;
 468 ;; [vclzq_u, vclzq_s])
 469 ;;
 470 (define_insn "mve_vclzq_<supf><mode>"
 471   [
 472    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 473         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 474          VCLZQ))
 475   ]
 476   "TARGET_HAVE_MVE"
 477   "vclz.i%#<V_sz_elem>  %q0, %q1"
 478   [(set_attr "type" "mve_move")
 479 ])
 480
 481 ;;
 482 ;; [vclsq_s])
 483 ;;
 484 (define_insn "mve_vclsq_s<mode>"
 485   [
 486    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 487         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 488          VCLSQ_S))
 489   ]
 490   "TARGET_HAVE_MVE"
 491   "vcls.s%#<V_sz_elem>  %q0, %q1"
 492   [(set_attr "type" "mve_move")
 493 ])
 494
 495 ;;
 496 ;; [vaddvq_s, vaddvq_u])
 497 ;;
 498 (define_insn "mve_vaddvq_<supf><mode>"
 499   [
 500    (set (match_operand:SI 0 "s_register_operand" "=Te")
 501         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 502          VADDVQ))
 503   ]
 504   "TARGET_HAVE_MVE"
 505   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 506   [(set_attr "type" "mve_move")
 507 ])
 508
 509 ;;
 510 ;; [vabsq_s])
 511 ;;
 512 (define_insn "mve_vabsq_s<mode>"
 513   [
 514    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 515         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 516          VABSQ_S))
 517   ]
 518   "TARGET_HAVE_MVE"
 519   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 520   [(set_attr "type" "mve_move")
 521 ])
 522
 523 ;;
 524 ;; [vrev32q_u, vrev32q_s])
 525 ;;
 526 (define_insn "mve_vrev32q_<supf><mode>"
 527   [
 528    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 529         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 530          VREV32Q))
 531   ]
 532   "TARGET_HAVE_MVE"
 533   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 534   [(set_attr "type" "mve_move")
 535 ])
 536
 537 ;;
 538 ;; [vmovltq_u, vmovltq_s])
 539 ;;
 540 (define_insn "mve_vmovltq_<supf><mode>"
 541   [
 542    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 543         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 544          VMOVLTQ))
 545   ]
 546   "TARGET_HAVE_MVE"
 547   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 548   [(set_attr "type" "mve_move")
 549 ])
 550
 551 ;;
 552 ;; [vmovlbq_s, vmovlbq_u])
 553 ;;
 554 (define_insn "mve_vmovlbq_<supf><mode>"
 555   [
 556    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 557         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 558          VMOVLBQ))
 559   ]
 560   "TARGET_HAVE_MVE"
 561   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 562   [(set_attr "type" "mve_move")
 563 ])
 564
 565 ;;
 566 ;; [vcvtpq_s, vcvtpq_u])
 567 ;;
 568 (define_insn "mve_vcvtpq_<supf><mode>"
 569   [
 570    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 571         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 572          VCVTPQ))
 573   ]
 574   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 575   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 576   [(set_attr "type" "mve_move")
 577 ])
 578
 579 ;;
 580 ;; [vcvtnq_s, vcvtnq_u])
 581 ;;
 582 (define_insn "mve_vcvtnq_<supf><mode>"
 583   [
 584    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 585         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 586          VCVTNQ))
 587   ]
 588   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 589   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 590   [(set_attr "type" "mve_move")
 591 ])
 592
 593 ;;
 594 ;; [vcvtmq_s, vcvtmq_u])
 595 ;;
 596 (define_insn "mve_vcvtmq_<supf><mode>"
 597   [
 598    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 599         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 600          VCVTMQ))
 601   ]
 602   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 603   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 604   [(set_attr "type" "mve_move")
 605 ])
 606
 607 ;;
 608 ;; [vcvtaq_u, vcvtaq_s])
 609 ;;
 610 (define_insn "mve_vcvtaq_<supf><mode>"
 611   [
 612    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 613         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 614          VCVTAQ))
 615   ]
 616   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 617   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 618   [(set_attr "type" "mve_move")
 619 ])
 620
 621 ;;
 622 ;; [vmvnq_n_u, vmvnq_n_s])
 623 ;;
 624 (define_insn "mve_vmvnq_n_<supf><mode>"
 625   [
 626    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 627         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 628          VMVNQ_N))
 629   ]
 630   "TARGET_HAVE_MVE"
 631   "vmvn.i%#<V_sz_elem>  %q0, %1"
 632   [(set_attr "type" "mve_move")
 633 ])
 634
 635 ;;
 636 ;; [vrev16q_u, vrev16q_s])
 637 ;;
 638 (define_insn "mve_vrev16q_<supf>v16qi"
 639   [
 640    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 641         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 642          VREV16Q))
 643   ]
 644   "TARGET_HAVE_MVE"
 645   "vrev16.8 %q0, %q1"
 646   [(set_attr "type" "mve_move")
 647 ])
 648
 649 ;;
 650 ;; [vaddlvq_s vaddlvq_u])
 651 ;;
 652 (define_insn "mve_vaddlvq_<supf>v4si"
 653   [
 654    (set (match_operand:DI 0 "s_register_operand" "=r")
 655         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 656          VADDLVQ))
 657   ]
 658   "TARGET_HAVE_MVE"
 659   "vaddlv.<supf>32 %Q0, %R0, %q1"
 660   [(set_attr "type" "mve_move")
 661 ])
 662
 663 ;;
 664 ;; [vctp8q vctp16q vctp32q vctp64q])
 665 ;;
 666 (define_insn "mve_vctp<mode1>qhi"
 667   [
 668    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 669         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 670         VCTPQ))
 671   ]
 672   "TARGET_HAVE_MVE"
 673   "vctp.<mode1> %1"
 674   [(set_attr "type" "mve_move")
 675 ])
 676
 677 ;;
 678 ;; [vpnot])
 679 ;;
 680 (define_insn "mve_vpnothi"
 681   [
 682    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 683         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 684          VPNOT))
 685   ]
 686   "TARGET_HAVE_MVE"
 687   "vpnot"
 688   [(set_attr "type" "mve_move")
 689 ])
 690
 691 ;;
 692 ;; [vsubq_n_f])
 693 ;;
 694 (define_insn "mve_vsubq_n_f<mode>"
 695   [
 696    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 697         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 698                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 699          VSUBQ_N_F))
 700   ]
 701   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 702   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 703   [(set_attr "type" "mve_move")
 704 ])
 705
 706 ;;
 707 ;; [vbrsrq_n_f])
 708 ;;
 709 (define_insn "mve_vbrsrq_n_f<mode>"
 710   [
 711    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 712         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 713                        (match_operand:SI 2 "s_register_operand" "r")]
 714          VBRSRQ_N_F))
 715   ]
 716   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 717   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 718   [(set_attr "type" "mve_move")
 719 ])
 720
 721 ;;
 722 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 723 ;;
 724 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 725   [
 726    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 727         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 728                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 729          VCVTQ_N_TO_F))
 730   ]
 731   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 732   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 733   [(set_attr "type" "mve_move")
 734 ])
 735
 736 ;; [vcreateq_f])
 737 ;;
 738 (define_insn "mve_vcreateq_f<mode>"
 739   [
 740    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 741         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 742                        (match_operand:DI 2 "s_register_operand" "r")]
 743          VCREATEQ_F))
 744   ]
 745   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 746   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 747   [(set_attr "type" "mve_move")
 748    (set_attr "length""8")])
 749
 750 ;;
 751 ;; [vcreateq_u, vcreateq_s])
 752 ;;
 753 (define_insn "mve_vcreateq_<supf><mode>"
 754   [
 755    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 756         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 757                        (match_operand:DI 2 "s_register_operand" "r")]
 758          VCREATEQ))
 759   ]
 760   "TARGET_HAVE_MVE"
 761   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 762   [(set_attr "type" "mve_move")
 763    (set_attr "length""8")])
 764
 765 ;;
 766 ;; [vshrq_n_s, vshrq_n_u])
 767 ;;
 768 (define_insn "mve_vshrq_n_<supf><mode>"
 769   [
 770    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 771         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 772                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 773          VSHRQ_N))
 774   ]
 775   "TARGET_HAVE_MVE"
 776   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 777   [(set_attr "type" "mve_move")
 778 ])
 779
 780 ;;
 781 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 782 ;;
 783 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 784   [
 785    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 786         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 787                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 788          VCVTQ_N_FROM_F))
 789   ]
 790   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 791   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 792   [(set_attr "type" "mve_move")
 793 ])
 794
 795 ;;
 796 ;; [vaddlvq_p_s])
 797 ;;
 798 (define_insn "mve_vaddlvq_p_<supf>v4si"
 799   [
 800    (set (match_operand:DI 0 "s_register_operand" "=r")
 801         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 802                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 803          VADDLVQ_P))
 804   ]
 805   "TARGET_HAVE_MVE"
 806   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 807   [(set_attr "type" "mve_move")
 808    (set_attr "length""8")])
 809
 810 ;;
 811 ;; [vcmpneq_u, vcmpneq_s])
 812 ;;
 813 (define_insn "mve_vcmpneq_<supf><mode>"
 814   [
 815    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 816         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 817                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 818          VCMPNEQ))
 819   ]
 820   "TARGET_HAVE_MVE"
 821   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
 822   [(set_attr "type" "mve_move")
 823 ])
 824
 825 ;;
 826 ;; [vshlq_s, vshlq_u])
 827 ;;
 828 (define_insn "mve_vshlq_<supf><mode>"
 829   [
 830    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 831         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 832                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 833          VSHLQ))
 834   ]
 835   "TARGET_HAVE_MVE"
 836   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
 837   [(set_attr "type" "mve_move")
 838 ])
 839
 840 ;;
 841 ;; [vabdq_s, vabdq_u])
 842 ;;
 843 (define_insn "mve_vabdq_<supf><mode>"
 844   [
 845    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 846         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 847                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 848          VABDQ))
 849   ]
 850   "TARGET_HAVE_MVE"
 851   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 852   [(set_attr "type" "mve_move")
 853 ])
 854
 855 ;;
 856 ;; [vaddq_n_s, vaddq_n_u])
 857 ;;
 858 (define_insn "mve_vaddq_n_<supf><mode>"
 859   [
 860    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 861         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 862                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 863          VADDQ_N))
 864   ]
 865   "TARGET_HAVE_MVE"
 866   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 867   [(set_attr "type" "mve_move")
 868 ])
 869
 870 ;;
 871 ;; [vaddvaq_s, vaddvaq_u])
 872 ;;
 873 (define_insn "mve_vaddvaq_<supf><mode>"
 874   [
 875    (set (match_operand:SI 0 "s_register_operand" "=Te")
 876         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 877                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 878          VADDVAQ))
 879   ]
 880   "TARGET_HAVE_MVE"
 881   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 882   [(set_attr "type" "mve_move")
 883 ])
 884
 885 ;;
 886 ;; [vaddvq_p_u, vaddvq_p_s])
 887 ;;
 888 (define_insn "mve_vaddvq_p_<supf><mode>"
 889   [
 890    (set (match_operand:SI 0 "s_register_operand" "=Te")
 891         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 892                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 893          VADDVQ_P))
 894   ]
 895   "TARGET_HAVE_MVE"
 896   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 897   [(set_attr "type" "mve_move")
 898    (set_attr "length""8")])
 899
 900 ;;
 901 ;; [vandq_u, vandq_s])
 902 ;;
 903 ;; signed and unsigned versions are the same: define the unsigned
 904 ;; insn, and use an expander for the signed one as we still reference
 905 ;; both names from arm_mve.h.
 906 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 907 (define_insn "mve_vandq_u<mode>"
 908   [
 909    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 910         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 911                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 912   ]
 913   "TARGET_HAVE_MVE"
 914   "@
 915    vand\t%q0, %q1, %q2
 916    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 917   [(set_attr "type" "mve_move")
 918 ])
 919 (define_expand "mve_vandq_s<mode>"
 920   [
 921    (set (match_operand:MVE_2 0 "s_register_operand")
 922         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 923                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 924   ]
 925   "TARGET_HAVE_MVE"
 926 )
 927
 928 ;;
 929 ;; [vbicq_s, vbicq_u])
 930 ;;
 931 (define_insn "mve_vbicq_u<mode>"
 932   [
 933    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 934         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 935                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 936   ]
 937   "TARGET_HAVE_MVE"
 938   "vbic\t%q0, %q1, %q2"
 939   [(set_attr "type" "mve_move")
 940 ])
 941
 942 (define_expand "mve_vbicq_s<mode>"
 943   [
 944    (set (match_operand:MVE_2 0 "s_register_operand")
 945         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 946                    (match_operand:MVE_2 1 "s_register_operand")))
 947   ]
 948   "TARGET_HAVE_MVE"
 949 )
 950
 951 ;;
 952 ;; [vbrsrq_n_u, vbrsrq_n_s])
 953 ;;
 954 (define_insn "mve_vbrsrq_n_<supf><mode>"
 955   [
 956    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 957         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 958                        (match_operand:SI 2 "s_register_operand" "r")]
 959          VBRSRQ_N))
 960   ]
 961   "TARGET_HAVE_MVE"
 962   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 963   [(set_attr "type" "mve_move")
 964 ])
 965
 966 ;;
 967 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
 968 ;;
 969 (define_insn "mve_vcaddq_rot270_<supf><mode>"
 970   [
 971    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 972         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 973                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 974          VCADDQ_ROT270))
 975   ]
 976   "TARGET_HAVE_MVE"
 977   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
 978   [(set_attr "type" "mve_move")
 979 ])
 980
 981 ;;
 982 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
 983 ;;
 984 (define_insn "mve_vcaddq_rot90_<supf><mode>"
 985   [
 986    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 987         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 988                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 989          VCADDQ_ROT90))
 990   ]
 991   "TARGET_HAVE_MVE"
 992   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
 993   [(set_attr "type" "mve_move")
 994 ])
 995
 996 ;;
 997 ;; [vcmpcsq_n_u])
 998 ;;
 999 (define_insn "mve_vcmpcsq_n_u<mode>"
1000   [
1001    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1002         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1003                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1004          VCMPCSQ_N_U))
1005   ]
1006   "TARGET_HAVE_MVE"
1007   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1008   [(set_attr "type" "mve_move")
1009 ])
1010
1011 ;;
1012 ;; [vcmpcsq_u])
1013 ;;
1014 (define_insn "mve_vcmpcsq_u<mode>"
1015   [
1016    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1017         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1018                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1019          VCMPCSQ_U))
1020   ]
1021   "TARGET_HAVE_MVE"
1022   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1023   [(set_attr "type" "mve_move")
1024 ])
1025
1026 ;;
1027 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1028 ;;
1029 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1030   [
1031    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1032         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1033                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1034          VCMPEQQ_N))
1035   ]
1036   "TARGET_HAVE_MVE"
1037   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1038   [(set_attr "type" "mve_move")
1039 ])
1040
1041 ;;
1042 ;; [vcmpeqq_u, vcmpeqq_s])
1043 ;;
1044 (define_insn "mve_vcmpeqq_<supf><mode>"
1045   [
1046    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1047         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1048                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1049          VCMPEQQ))
1050   ]
1051   "TARGET_HAVE_MVE"
1052   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1053   [(set_attr "type" "mve_move")
1054 ])
1055
1056 ;;
1057 ;; [vcmpgeq_n_s])
1058 ;;
1059 (define_insn "mve_vcmpgeq_n_s<mode>"
1060   [
1061    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1062         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1063                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1064          VCMPGEQ_N_S))
1065   ]
1066   "TARGET_HAVE_MVE"
1067   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1068   [(set_attr "type" "mve_move")
1069 ])
1070
1071 ;;
1072 ;; [vcmpgeq_s])
1073 ;;
1074 (define_insn "mve_vcmpgeq_s<mode>"
1075   [
1076    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1077         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1078                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1079          VCMPGEQ_S))
1080   ]
1081   "TARGET_HAVE_MVE"
1082   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1083   [(set_attr "type" "mve_move")
1084 ])
1085
1086 ;;
1087 ;; [vcmpgtq_n_s])
1088 ;;
1089 (define_insn "mve_vcmpgtq_n_s<mode>"
1090   [
1091    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1092         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1093                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1094          VCMPGTQ_N_S))
1095   ]
1096   "TARGET_HAVE_MVE"
1097   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1098   [(set_attr "type" "mve_move")
1099 ])
1100
1101 ;;
1102 ;; [vcmpgtq_s])
1103 ;;
1104 (define_insn "mve_vcmpgtq_s<mode>"
1105   [
1106    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1107         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1108                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1109          VCMPGTQ_S))
1110   ]
1111   "TARGET_HAVE_MVE"
1112   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1113   [(set_attr "type" "mve_move")
1114 ])
1115
1116 ;;
1117 ;; [vcmphiq_n_u])
1118 ;;
1119 (define_insn "mve_vcmphiq_n_u<mode>"
1120   [
1121    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1122         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1123                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1124          VCMPHIQ_N_U))
1125   ]
1126   "TARGET_HAVE_MVE"
1127   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1128   [(set_attr "type" "mve_move")
1129 ])
1130
1131 ;;
1132 ;; [vcmphiq_u])
1133 ;;
1134 (define_insn "mve_vcmphiq_u<mode>"
1135   [
1136    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1137         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1138                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1139          VCMPHIQ_U))
1140   ]
1141   "TARGET_HAVE_MVE"
1142   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1143   [(set_attr "type" "mve_move")
1144 ])
1145
1146 ;;
1147 ;; [vcmpleq_n_s])
1148 ;;
1149 (define_insn "mve_vcmpleq_n_s<mode>"
1150   [
1151    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1152         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1153                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1154          VCMPLEQ_N_S))
1155   ]
1156   "TARGET_HAVE_MVE"
1157   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1158   [(set_attr "type" "mve_move")
1159 ])
1160
1161 ;;
1162 ;; [vcmpleq_s])
1163 ;;
1164 (define_insn "mve_vcmpleq_s<mode>"
1165   [
1166    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1167         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1168                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1169          VCMPLEQ_S))
1170   ]
1171   "TARGET_HAVE_MVE"
1172   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1173   [(set_attr "type" "mve_move")
1174 ])
1175
1176 ;;
1177 ;; [vcmpltq_n_s])
1178 ;;
1179 (define_insn "mve_vcmpltq_n_s<mode>"
1180   [
1181    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1182         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1183                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1184          VCMPLTQ_N_S))
1185   ]
1186   "TARGET_HAVE_MVE"
1187   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1188   [(set_attr "type" "mve_move")
1189 ])
1190
1191 ;;
1192 ;; [vcmpltq_s])
1193 ;;
1194 (define_insn "mve_vcmpltq_s<mode>"
1195   [
1196    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1197         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1198                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1199          VCMPLTQ_S))
1200   ]
1201   "TARGET_HAVE_MVE"
1202   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1203   [(set_attr "type" "mve_move")
1204 ])
1205
1206 ;;
1207 ;; [vcmpneq_n_u, vcmpneq_n_s])
1208 ;;
1209 (define_insn "mve_vcmpneq_n_<supf><mode>"
1210   [
1211    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1212         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1213                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1214          VCMPNEQ_N))
1215   ]
1216   "TARGET_HAVE_MVE"
1217   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1218   [(set_attr "type" "mve_move")
1219 ])
1220
1221 ;;
1222 ;; [veorq_u, veorq_s])
1223 ;;
1224 (define_insn "mve_veorq_u<mode>"
1225   [
1226    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1227         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1228                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1229   ]
1230   "TARGET_HAVE_MVE"
1231   "veor\t%q0, %q1, %q2"
1232   [(set_attr "type" "mve_move")
1233 ])
1234 (define_expand "mve_veorq_s<mode>"
1235   [
1236    (set (match_operand:MVE_2 0 "s_register_operand")
1237         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1238                    (match_operand:MVE_2 2 "s_register_operand")))
1239   ]
1240   "TARGET_HAVE_MVE"
1241 )
1242
1243 ;;
1244 ;; [vhaddq_n_u, vhaddq_n_s])
1245 ;;
1246 (define_insn "mve_vhaddq_n_<supf><mode>"
1247   [
1248    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1249         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1250                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1251          VHADDQ_N))
1252   ]
1253   "TARGET_HAVE_MVE"
1254   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1255   [(set_attr "type" "mve_move")
1256 ])
1257
1258 ;;
1259 ;; [vhaddq_s, vhaddq_u])
1260 ;;
1261 (define_insn "mve_vhaddq_<supf><mode>"
1262   [
1263    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1264         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1265                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1266          VHADDQ))
1267   ]
1268   "TARGET_HAVE_MVE"
1269   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1270   [(set_attr "type" "mve_move")
1271 ])
1272
1273 ;;
1274 ;; [vhcaddq_rot270_s])
1275 ;;
1276 (define_insn "mve_vhcaddq_rot270_s<mode>"
1277   [
1278    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1279         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1280                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1281          VHCADDQ_ROT270_S))
1282   ]
1283   "TARGET_HAVE_MVE"
1284   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1285   [(set_attr "type" "mve_move")
1286 ])
1287
1288 ;;
1289 ;; [vhcaddq_rot90_s])
1290 ;;
1291 (define_insn "mve_vhcaddq_rot90_s<mode>"
1292   [
1293    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1294         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1295                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1296          VHCADDQ_ROT90_S))
1297   ]
1298   "TARGET_HAVE_MVE"
1299   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1300   [(set_attr "type" "mve_move")
1301 ])
1302
1303 ;;
1304 ;; [vhsubq_n_u, vhsubq_n_s])
1305 ;;
1306 (define_insn "mve_vhsubq_n_<supf><mode>"
1307   [
1308    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1309         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1310                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1311          VHSUBQ_N))
1312   ]
1313   "TARGET_HAVE_MVE"
1314   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1315   [(set_attr "type" "mve_move")
1316 ])
1317
1318 ;;
1319 ;; [vhsubq_s, vhsubq_u])
1320 ;;
1321 (define_insn "mve_vhsubq_<supf><mode>"
1322   [
1323    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1324         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1325                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1326          VHSUBQ))
1327   ]
1328   "TARGET_HAVE_MVE"
1329   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1330   [(set_attr "type" "mve_move")
1331 ])
1332
1333 ;;
1334 ;; [vmaxaq_s])
1335 ;;
1336 (define_insn "mve_vmaxaq_s<mode>"
1337   [
1338    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1339         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1340                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1341          VMAXAQ_S))
1342   ]
1343   "TARGET_HAVE_MVE"
1344   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1345   [(set_attr "type" "mve_move")
1346 ])
1347
1348 ;;
1349 ;; [vmaxavq_s])
1350 ;;
1351 (define_insn "mve_vmaxavq_s<mode>"
1352   [
1353    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1354         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1355                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1356          VMAXAVQ_S))
1357   ]
1358   "TARGET_HAVE_MVE"
1359   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1360   [(set_attr "type" "mve_move")
1361 ])
1362
1363 ;;
1364 ;; [vmaxq_u, vmaxq_s])
1365 ;;
1366 (define_insn "mve_vmaxq_s<mode>"
1367   [
1368    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1369         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1370                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1371   ]
1372   "TARGET_HAVE_MVE"
1373   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1374   [(set_attr "type" "mve_move")
1375 ])
1376
1377 (define_insn "mve_vmaxq_u<mode>"
1378   [
1379    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1380         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1381                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1382   ]
1383   "TARGET_HAVE_MVE"
1384   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1385   [(set_attr "type" "mve_move")
1386 ])
1387
1388 ;;
1389 ;; [vmaxvq_u, vmaxvq_s])
1390 ;;
1391 (define_insn "mve_vmaxvq_<supf><mode>"
1392   [
1393    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1394         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1395                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1396          VMAXVQ))
1397   ]
1398   "TARGET_HAVE_MVE"
1399   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1400   [(set_attr "type" "mve_move")
1401 ])
1402
1403 ;;
1404 ;; [vminaq_s])
1405 ;;
1406 (define_insn "mve_vminaq_s<mode>"
1407   [
1408    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1409         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1410                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1411          VMINAQ_S))
1412   ]
1413   "TARGET_HAVE_MVE"
1414   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1415   [(set_attr "type" "mve_move")
1416 ])
1417
1418 ;;
1419 ;; [vminavq_s])
1420 ;;
1421 (define_insn "mve_vminavq_s<mode>"
1422   [
1423    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1424         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1425                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1426          VMINAVQ_S))
1427   ]
1428   "TARGET_HAVE_MVE"
1429   "vminav.s%#<V_sz_elem>\t%0, %q2"
1430   [(set_attr "type" "mve_move")
1431 ])
1432
1433 ;;
1434 ;; [vminq_s, vminq_u])
1435 ;;
1436 (define_insn "mve_vminq_s<mode>"
1437   [
1438    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1439         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1440                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1441   ]
1442   "TARGET_HAVE_MVE"
1443   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1444   [(set_attr "type" "mve_move")
1445 ])
1446
1447 (define_insn "mve_vminq_u<mode>"
1448   [
1449    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1450         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1451                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1452   ]
1453   "TARGET_HAVE_MVE"
1454   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1455   [(set_attr "type" "mve_move")
1456 ])
1457
1458 ;;
1459 ;; [vminvq_u, vminvq_s])
1460 ;;
1461 (define_insn "mve_vminvq_<supf><mode>"
1462   [
1463    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1464         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1465                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1466          VMINVQ))
1467   ]
1468   "TARGET_HAVE_MVE"
1469   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1470   [(set_attr "type" "mve_move")
1471 ])
1472
1473 ;;
1474 ;; [vmladavq_u, vmladavq_s])
1475 ;;
1476 (define_insn "mve_vmladavq_<supf><mode>"
1477   [
1478    (set (match_operand:SI 0 "s_register_operand" "=Te")
1479         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1480                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1481          VMLADAVQ))
1482   ]
1483   "TARGET_HAVE_MVE"
1484   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1485   [(set_attr "type" "mve_move")
1486 ])
1487
1488 ;;
1489 ;; [vmladavxq_s])
1490 ;;
1491 (define_insn "mve_vmladavxq_s<mode>"
1492   [
1493    (set (match_operand:SI 0 "s_register_operand" "=Te")
1494         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1495                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1496          VMLADAVXQ_S))
1497   ]
1498   "TARGET_HAVE_MVE"
1499   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1500   [(set_attr "type" "mve_move")
1501 ])
1502
1503 ;;
1504 ;; [vmlsdavq_s])
1505 ;;
1506 (define_insn "mve_vmlsdavq_s<mode>"
1507   [
1508    (set (match_operand:SI 0 "s_register_operand" "=Te")
1509         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1510                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1511          VMLSDAVQ_S))
1512   ]
1513   "TARGET_HAVE_MVE"
1514   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1515   [(set_attr "type" "mve_move")
1516 ])
1517
1518 ;;
1519 ;; [vmlsdavxq_s])
1520 ;;
1521 (define_insn "mve_vmlsdavxq_s<mode>"
1522   [
1523    (set (match_operand:SI 0 "s_register_operand" "=Te")
1524         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1525                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1526          VMLSDAVXQ_S))
1527   ]
1528   "TARGET_HAVE_MVE"
1529   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1530   [(set_attr "type" "mve_move")
1531 ])
1532
1533 ;;
1534 ;; [vmulhq_s, vmulhq_u])
1535 ;;
1536 (define_insn "mve_vmulhq_<supf><mode>"
1537   [
1538    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1539         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1540                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1541          VMULHQ))
1542   ]
1543   "TARGET_HAVE_MVE"
1544   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1545   [(set_attr "type" "mve_move")
1546 ])
1547
1548 ;;
1549 ;; [vmullbq_int_u, vmullbq_int_s])
1550 ;;
1551 (define_insn "mve_vmullbq_int_<supf><mode>"
1552   [
1553    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1554         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1555                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1556          VMULLBQ_INT))
1557   ]
1558   "TARGET_HAVE_MVE"
1559   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1560   [(set_attr "type" "mve_move")
1561 ])
1562
1563 ;;
1564 ;; [vmulltq_int_u, vmulltq_int_s])
1565 ;;
1566 (define_insn "mve_vmulltq_int_<supf><mode>"
1567   [
1568    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1569         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1570                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1571          VMULLTQ_INT))
1572   ]
1573   "TARGET_HAVE_MVE"
1574   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1575   [(set_attr "type" "mve_move")
1576 ])
1577
1578 ;;
1579 ;; [vmulq_n_u, vmulq_n_s])
1580 ;;
1581 (define_insn "mve_vmulq_n_<supf><mode>"
1582   [
1583    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1584         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1585                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1586          VMULQ_N))
1587   ]
1588   "TARGET_HAVE_MVE"
1589   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1590   [(set_attr "type" "mve_move")
1591 ])
1592
1593 ;;
1594 ;; [vmulq_u, vmulq_s])
1595 ;;
1596 (define_insn "mve_vmulq_<supf><mode>"
1597   [
1598    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1599         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1600                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1601          VMULQ))
1602   ]
1603   "TARGET_HAVE_MVE"
1604   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1605   [(set_attr "type" "mve_move")
1606 ])
1607
1608 (define_insn "mve_vmulq<mode>"
1609   [
1610    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1611         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1612                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1613   ]
1614   "TARGET_HAVE_MVE"
1615   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1616   [(set_attr "type" "mve_move")
1617 ])
1618
1619 ;;
1620 ;; [vornq_u, vornq_s])
1621 ;;
1622 (define_insn "mve_vornq_<supf><mode>"
1623   [
1624    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1625         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1626                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1627          VORNQ))
1628   ]
1629   "TARGET_HAVE_MVE"
1630   "vorn %q0, %q1, %q2"
1631   [(set_attr "type" "mve_move")
1632 ])
1633
1634 ;;
1635 ;; [vorrq_s, vorrq_u])
1636 ;;
1637 ;; signed and unsigned versions are the same: define the unsigned
1638 ;; insn, and use an expander for the signed one as we still reference
1639 ;; both names from arm_mve.h.
1640 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1641 (define_insn "mve_vorrq_s<mode>"
1642   [
1643    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1644         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1645                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1646   ]
1647   "TARGET_HAVE_MVE"
1648   "@
1649    vorr\t%q0, %q1, %q2
1650    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1651   [(set_attr "type" "mve_move")
1652 ])
1653 (define_expand "mve_vorrq_u<mode>"
1654   [
1655    (set (match_operand:MVE_2 0 "s_register_operand")
1656         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1657                    (match_operand:MVE_2 2 "neon_logic_op2")))
1658   ]
1659   "TARGET_HAVE_MVE"
1660 )
1661
1662 ;;
1663 ;; [vqaddq_n_s, vqaddq_n_u])
1664 ;;
1665 (define_insn "mve_vqaddq_n_<supf><mode>"
1666   [
1667    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1668         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1669                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1670          VQADDQ_N))
1671   ]
1672   "TARGET_HAVE_MVE"
1673   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1674   [(set_attr "type" "mve_move")
1675 ])
1676
1677 ;;
1678 ;; [vqaddq_u, vqaddq_s])
1679 ;;
1680 (define_insn "mve_vqaddq_<supf><mode>"
1681   [
1682    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1683         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1684                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1685          VQADDQ))
1686   ]
1687   "TARGET_HAVE_MVE"
1688   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1689   [(set_attr "type" "mve_move")
1690 ])
1691
1692 ;;
1693 ;; [vqdmulhq_n_s])
1694 ;;
1695 (define_insn "mve_vqdmulhq_n_s<mode>"
1696   [
1697    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1698         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1699                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1700          VQDMULHQ_N_S))
1701   ]
1702   "TARGET_HAVE_MVE"
1703   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1704   [(set_attr "type" "mve_move")
1705 ])
1706
1707 ;;
1708 ;; [vqdmulhq_s])
1709 ;;
1710 (define_insn "mve_vqdmulhq_s<mode>"
1711   [
1712    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1713         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1714                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1715          VQDMULHQ_S))
1716   ]
1717   "TARGET_HAVE_MVE"
1718   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1719   [(set_attr "type" "mve_move")
1720 ])
1721
1722 ;;
1723 ;; [vqrdmulhq_n_s])
1724 ;;
1725 (define_insn "mve_vqrdmulhq_n_s<mode>"
1726   [
1727    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1728         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1729                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1730          VQRDMULHQ_N_S))
1731   ]
1732   "TARGET_HAVE_MVE"
1733   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1734   [(set_attr "type" "mve_move")
1735 ])
1736
1737 ;;
1738 ;; [vqrdmulhq_s])
1739 ;;
1740 (define_insn "mve_vqrdmulhq_s<mode>"
1741   [
1742    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1743         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1744                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1745          VQRDMULHQ_S))
1746   ]
1747   "TARGET_HAVE_MVE"
1748   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1749   [(set_attr "type" "mve_move")
1750 ])
1751
1752 ;;
1753 ;; [vqrshlq_n_s, vqrshlq_n_u])
1754 ;;
1755 (define_insn "mve_vqrshlq_n_<supf><mode>"
1756   [
1757    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1758         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1759                        (match_operand:SI 2 "s_register_operand" "r")]
1760          VQRSHLQ_N))
1761   ]
1762   "TARGET_HAVE_MVE"
1763   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1764   [(set_attr "type" "mve_move")
1765 ])
1766
1767 ;;
1768 ;; [vqrshlq_s, vqrshlq_u])
1769 ;;
1770 (define_insn "mve_vqrshlq_<supf><mode>"
1771   [
1772    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1773         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1774                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1775          VQRSHLQ))
1776   ]
1777   "TARGET_HAVE_MVE"
1778   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1779   [(set_attr "type" "mve_move")
1780 ])
1781
1782 ;;
1783 ;; [vqshlq_n_s, vqshlq_n_u])
1784 ;;
1785 (define_insn "mve_vqshlq_n_<supf><mode>"
1786   [
1787    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1788         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1789                        (match_operand:SI 2 "immediate_operand" "i")]
1790          VQSHLQ_N))
1791   ]
1792   "TARGET_HAVE_MVE"
1793   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1794   [(set_attr "type" "mve_move")
1795 ])
1796
1797 ;;
1798 ;; [vqshlq_r_u, vqshlq_r_s])
1799 ;;
1800 (define_insn "mve_vqshlq_r_<supf><mode>"
1801   [
1802    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1803         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1804                        (match_operand:SI 2 "s_register_operand" "r")]
1805          VQSHLQ_R))
1806   ]
1807   "TARGET_HAVE_MVE"
1808   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1809   [(set_attr "type" "mve_move")
1810 ])
1811
1812 ;;
1813 ;; [vqshlq_s, vqshlq_u])
1814 ;;
1815 (define_insn "mve_vqshlq_<supf><mode>"
1816   [
1817    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1818         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1819                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1820          VQSHLQ))
1821   ]
1822   "TARGET_HAVE_MVE"
1823   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1824   [(set_attr "type" "mve_move")
1825 ])
1826
1827 ;;
1828 ;; [vqshluq_n_s])
1829 ;;
1830 (define_insn "mve_vqshluq_n_s<mode>"
1831   [
1832    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1833         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1834                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1835          VQSHLUQ_N_S))
1836   ]
1837   "TARGET_HAVE_MVE"
1838   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1839   [(set_attr "type" "mve_move")
1840 ])
1841
1842 ;;
1843 ;; [vqsubq_n_s, vqsubq_n_u])
1844 ;;
1845 (define_insn "mve_vqsubq_n_<supf><mode>"
1846   [
1847    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1848         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1849                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1850          VQSUBQ_N))
1851   ]
1852   "TARGET_HAVE_MVE"
1853   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1854   [(set_attr "type" "mve_move")
1855 ])
1856
1857 ;;
1858 ;; [vqsubq_u, vqsubq_s])
1859 ;;
1860 (define_insn "mve_vqsubq_<supf><mode>"
1861   [
1862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1864                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1865          VQSUBQ))
1866   ]
1867   "TARGET_HAVE_MVE"
1868   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1869   [(set_attr "type" "mve_move")
1870 ])
1871
1872 ;;
1873 ;; [vrhaddq_s, vrhaddq_u])
1874 ;;
1875 (define_insn "mve_vrhaddq_<supf><mode>"
1876   [
1877    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1878         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1879                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1880          VRHADDQ))
1881   ]
1882   "TARGET_HAVE_MVE"
1883   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1884   [(set_attr "type" "mve_move")
1885 ])
1886
1887 ;;
1888 ;; [vrmulhq_s, vrmulhq_u])
1889 ;;
1890 (define_insn "mve_vrmulhq_<supf><mode>"
1891   [
1892    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1893         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1894                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1895          VRMULHQ))
1896   ]
1897   "TARGET_HAVE_MVE"
1898   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1899   [(set_attr "type" "mve_move")
1900 ])
1901
1902 ;;
1903 ;; [vrshlq_n_u, vrshlq_n_s])
1904 ;;
1905 (define_insn "mve_vrshlq_n_<supf><mode>"
1906   [
1907    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1908         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1909                        (match_operand:SI 2 "s_register_operand" "r")]
1910          VRSHLQ_N))
1911   ]
1912   "TARGET_HAVE_MVE"
1913   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1914   [(set_attr "type" "mve_move")
1915 ])
1916
1917 ;;
1918 ;; [vrshlq_s, vrshlq_u])
1919 ;;
1920 (define_insn "mve_vrshlq_<supf><mode>"
1921   [
1922    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1923         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1924                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1925          VRSHLQ))
1926   ]
1927   "TARGET_HAVE_MVE"
1928   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1929   [(set_attr "type" "mve_move")
1930 ])
1931
1932 ;;
1933 ;; [vrshrq_n_s, vrshrq_n_u])
1934 ;;
1935 (define_insn "mve_vrshrq_n_<supf><mode>"
1936   [
1937    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1938         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1939                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1940          VRSHRQ_N))
1941   ]
1942   "TARGET_HAVE_MVE"
1943   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1944   [(set_attr "type" "mve_move")
1945 ])
1946
1947 ;;
1948 ;; [vshlq_n_u, vshlq_n_s])
1949 ;;
1950 (define_insn "mve_vshlq_n_<supf><mode>"
1951   [
1952    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1953         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1954                        (match_operand:SI 2 "immediate_operand" "i")]
1955          VSHLQ_N))
1956   ]
1957   "TARGET_HAVE_MVE"
1958   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1959   [(set_attr "type" "mve_move")
1960 ])
1961
1962 ;;
1963 ;; [vshlq_r_s, vshlq_r_u])
1964 ;;
1965 (define_insn "mve_vshlq_r_<supf><mode>"
1966   [
1967    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1968         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1969                        (match_operand:SI 2 "s_register_operand" "r")]
1970          VSHLQ_R))
1971   ]
1972   "TARGET_HAVE_MVE"
1973   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1974   [(set_attr "type" "mve_move")
1975 ])
1976
1977 ;;
1978 ;; [vsubq_n_s, vsubq_n_u])
1979 ;;
1980 (define_insn "mve_vsubq_n_<supf><mode>"
1981   [
1982    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1983         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1984                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1985          VSUBQ_N))
1986   ]
1987   "TARGET_HAVE_MVE"
1988   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1989   [(set_attr "type" "mve_move")
1990 ])
1991
1992 ;;
1993 ;; [vsubq_s, vsubq_u])
1994 ;;
1995 (define_insn "mve_vsubq_<supf><mode>"
1996   [
1997    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1998         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1999                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2000          VSUBQ))
2001   ]
2002   "TARGET_HAVE_MVE"
2003   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2004   [(set_attr "type" "mve_move")
2005 ])
2006
2007 (define_insn "mve_vsubq<mode>"
2008   [
2009    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2010         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
2011                      (match_operand:MVE_2 2 "s_register_operand" "w")))
2012   ]
2013   "TARGET_HAVE_MVE"
2014   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2015   [(set_attr "type" "mve_move")
2016 ])
2017
2018 ;;
2019 ;; [vabdq_f])
2020 ;;
2021 (define_insn "mve_vabdq_f<mode>"
2022   [
2023    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2024         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2025                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2026          VABDQ_F))
2027   ]
2028   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2029   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2030   [(set_attr "type" "mve_move")
2031 ])
2032
2033 ;;
2034 ;; [vaddlvaq_s vaddlvaq_u])
2035 ;;
2036 (define_insn "mve_vaddlvaq_<supf>v4si"
2037   [
2038    (set (match_operand:DI 0 "s_register_operand" "=r")
2039         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2040                     (match_operand:V4SI 2 "s_register_operand" "w")]
2041          VADDLVAQ))
2042   ]
2043   "TARGET_HAVE_MVE"
2044   "vaddlva.<supf>32 %Q0, %R0, %q2"
2045   [(set_attr "type" "mve_move")
2046 ])
2047
2048 ;;
2049 ;; [vaddq_n_f])
2050 ;;
2051 (define_insn "mve_vaddq_n_f<mode>"
2052   [
2053    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2054         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2055                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2056          VADDQ_N_F))
2057   ]
2058   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2059   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2060   [(set_attr "type" "mve_move")
2061 ])
2062
2063 ;;
2064 ;; [vandq_f])
2065 ;;
2066 (define_insn "mve_vandq_f<mode>"
2067   [
2068    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2069         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2070                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2071   ]
2072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2073   "vand %q0, %q1, %q2"
2074   [(set_attr "type" "mve_move")
2075 ])
2076
2077 ;;
2078 ;; [vbicq_f])
2079 ;;
2080 (define_insn "mve_vbicq_f<mode>"
2081   [
2082    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2083         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
2084                               (match_operand:MVE_0 2 "s_register_operand" "w")))
2085   ]
2086   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2087   "vbic %q0, %q1, %q2"
2088   [(set_attr "type" "mve_move")
2089 ])
2090
2091 ;;
2092 ;; [vbicq_n_s, vbicq_n_u])
2093 ;;
2094 (define_insn "mve_vbicq_n_<supf><mode>"
2095   [
2096    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2097         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2098                        (match_operand:SI 2 "immediate_operand" "i")]
2099          VBICQ_N))
2100   ]
2101   "TARGET_HAVE_MVE"
2102   "vbic.i%#<V_sz_elem>  %q0, %2"
2103   [(set_attr "type" "mve_move")
2104 ])
2105
2106 ;;
2107 ;; [vcaddq_rot270_f])
2108 ;;
2109 (define_insn "mve_vcaddq_rot270_f<mode>"
2110   [
2111    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2112         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2113                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2114          VCADDQ_ROT270_F))
2115   ]
2116   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2117   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2118   [(set_attr "type" "mve_move")
2119 ])
2120
2121 ;;
2122 ;; [vcaddq_rot90_f])
2123 ;;
2124 (define_insn "mve_vcaddq_rot90_f<mode>"
2125   [
2126    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2127         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2128                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2129          VCADDQ_ROT90_F))
2130   ]
2131   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2132   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2133   [(set_attr "type" "mve_move")
2134 ])
2135
2136 ;;
2137 ;; [vcmpeqq_f])
2138 ;;
2139 (define_insn "mve_vcmpeqq_f<mode>"
2140   [
2141    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2142         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2143                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2144          VCMPEQQ_F))
2145   ]
2146   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2147   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2148   [(set_attr "type" "mve_move")
2149 ])
2150
2151 ;;
2152 ;; [vcmpeqq_n_f])
2153 ;;
2154 (define_insn "mve_vcmpeqq_n_f<mode>"
2155   [
2156    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2157         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2158                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2159          VCMPEQQ_N_F))
2160   ]
2161   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2162   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2163   [(set_attr "type" "mve_move")
2164 ])
2165
2166 ;;
2167 ;; [vcmpgeq_f])
2168 ;;
2169 (define_insn "mve_vcmpgeq_f<mode>"
2170   [
2171    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2172         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2173                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2174          VCMPGEQ_F))
2175   ]
2176   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2177   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2178   [(set_attr "type" "mve_move")
2179 ])
2180
2181 ;;
2182 ;; [vcmpgeq_n_f])
2183 ;;
2184 (define_insn "mve_vcmpgeq_n_f<mode>"
2185   [
2186    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2187         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2188                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2189          VCMPGEQ_N_F))
2190   ]
2191   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2192   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2193   [(set_attr "type" "mve_move")
2194 ])
2195
2196 ;;
2197 ;; [vcmpgtq_f])
2198 ;;
2199 (define_insn "mve_vcmpgtq_f<mode>"
2200   [
2201    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2202         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2203                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2204          VCMPGTQ_F))
2205   ]
2206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2207   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2208   [(set_attr "type" "mve_move")
2209 ])
2210
2211 ;;
2212 ;; [vcmpgtq_n_f])
2213 ;;
2214 (define_insn "mve_vcmpgtq_n_f<mode>"
2215   [
2216    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2217         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2218                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2219          VCMPGTQ_N_F))
2220   ]
2221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2222   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2223   [(set_attr "type" "mve_move")
2224 ])
2225
2226 ;;
2227 ;; [vcmpleq_f])
2228 ;;
2229 (define_insn "mve_vcmpleq_f<mode>"
2230   [
2231    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2232         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2233                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2234          VCMPLEQ_F))
2235   ]
2236   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2237   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2238   [(set_attr "type" "mve_move")
2239 ])
2240
2241 ;;
2242 ;; [vcmpleq_n_f])
2243 ;;
2244 (define_insn "mve_vcmpleq_n_f<mode>"
2245   [
2246    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2247         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2248                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2249          VCMPLEQ_N_F))
2250   ]
2251   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2252   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2253   [(set_attr "type" "mve_move")
2254 ])
2255
2256 ;;
2257 ;; [vcmpltq_f])
2258 ;;
2259 (define_insn "mve_vcmpltq_f<mode>"
2260   [
2261    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2262         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2263                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2264          VCMPLTQ_F))
2265   ]
2266   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2267   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2268   [(set_attr "type" "mve_move")
2269 ])
2270
2271 ;;
2272 ;; [vcmpltq_n_f])
2273 ;;
2274 (define_insn "mve_vcmpltq_n_f<mode>"
2275   [
2276    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2277         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2278                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2279          VCMPLTQ_N_F))
2280   ]
2281   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2282   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2283   [(set_attr "type" "mve_move")
2284 ])
2285
2286 ;;
2287 ;; [vcmpneq_f])
2288 ;;
2289 (define_insn "mve_vcmpneq_f<mode>"
2290   [
2291    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2292         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2293                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2294          VCMPNEQ_F))
2295   ]
2296   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2297   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2298   [(set_attr "type" "mve_move")
2299 ])
2300
2301 ;;
2302 ;; [vcmpneq_n_f])
2303 ;;
2304 (define_insn "mve_vcmpneq_n_f<mode>"
2305   [
2306    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2307         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2308                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2309          VCMPNEQ_N_F))
2310   ]
2311   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2312   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2313   [(set_attr "type" "mve_move")
2314 ])
2315
2316 ;;
2317 ;; [vcmulq_f])
2318 ;;
2319 (define_insn "mve_vcmulq_f<mode>"
2320   [
2321    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2322         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2323                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2324          VCMULQ_F))
2325   ]
2326   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2327   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2328   [(set_attr "type" "mve_move")
2329 ])
2330
2331 ;;
2332 ;; [vcmulq_rot180_f])
2333 ;;
2334 (define_insn "mve_vcmulq_rot180_f<mode>"
2335   [
2336    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2337         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2338                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2339          VCMULQ_ROT180_F))
2340   ]
2341   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2342   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2343   [(set_attr "type" "mve_move")
2344 ])
2345
2346 ;;
2347 ;; [vcmulq_rot270_f])
2348 ;;
2349 (define_insn "mve_vcmulq_rot270_f<mode>"
2350   [
2351    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2352         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2353                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2354          VCMULQ_ROT270_F))
2355   ]
2356   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2357   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2358   [(set_attr "type" "mve_move")
2359 ])
2360
2361 ;;
2362 ;; [vcmulq_rot90_f])
2363 ;;
2364 (define_insn "mve_vcmulq_rot90_f<mode>"
2365   [
2366    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2367         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2368                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2369          VCMULQ_ROT90_F))
2370   ]
2371   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2372   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2373   [(set_attr "type" "mve_move")
2374 ])
2375
2376 ;;
2377 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2378 ;;
2379 (define_insn "mve_vctp<mode1>q_mhi"
2380   [
2381    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2382         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2383                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2384          VCTPQ_M))
2385   ]
2386   "TARGET_HAVE_MVE"
2387   "vpst\;vctpt.<mode1> %1"
2388   [(set_attr "type" "mve_move")
2389    (set_attr "length""8")])
2390
2391 ;;
2392 ;; [vcvtbq_f16_f32])
2393 ;;
2394 (define_insn "mve_vcvtbq_f16_f32v8hf"
2395   [
2396    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2397         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2398                       (match_operand:V4SF 2 "s_register_operand" "w")]
2399          VCVTBQ_F16_F32))
2400   ]
2401   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2402   "vcvtb.f16.f32 %q0, %q2"
2403   [(set_attr "type" "mve_move")
2404 ])
2405
2406 ;;
2407 ;; [vcvttq_f16_f32])
2408 ;;
2409 (define_insn "mve_vcvttq_f16_f32v8hf"
2410   [
2411    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2412         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2413                       (match_operand:V4SF 2 "s_register_operand" "w")]
2414          VCVTTQ_F16_F32))
2415   ]
2416   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2417   "vcvtt.f16.f32 %q0, %q2"
2418   [(set_attr "type" "mve_move")
2419 ])
2420
2421 ;;
2422 ;; [veorq_f])
2423 ;;
2424 (define_insn "mve_veorq_f<mode>"
2425   [
2426    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2427         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2428                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2429   ]
2430   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2431   "veor %q0, %q1, %q2"
2432   [(set_attr "type" "mve_move")
2433 ])
2434
2435 ;;
2436 ;; [vmaxnmaq_f])
2437 ;;
2438 (define_insn "mve_vmaxnmaq_f<mode>"
2439   [
2440    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2441         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2442                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2443          VMAXNMAQ_F))
2444   ]
2445   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2446   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2447   [(set_attr "type" "mve_move")
2448 ])
2449
2450 ;;
2451 ;; [vmaxnmavq_f])
2452 ;;
2453 (define_insn "mve_vmaxnmavq_f<mode>"
2454   [
2455    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2456         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2457                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2458          VMAXNMAVQ_F))
2459   ]
2460   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2461   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2462   [(set_attr "type" "mve_move")
2463 ])
2464
2465 ;;
2466 ;; [vmaxnmq_f])
2467 ;;
2468 (define_insn "mve_vmaxnmq_f<mode>"
2469   [
2470    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2471         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2472                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2473   ]
2474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2475   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2476   [(set_attr "type" "mve_move")
2477 ])
2478
2479 ;;
2480 ;; [vmaxnmvq_f])
2481 ;;
2482 (define_insn "mve_vmaxnmvq_f<mode>"
2483   [
2484    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2485         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2486                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2487          VMAXNMVQ_F))
2488   ]
2489   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2490   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2491   [(set_attr "type" "mve_move")
2492 ])
2493
2494 ;;
2495 ;; [vminnmaq_f])
2496 ;;
2497 (define_insn "mve_vminnmaq_f<mode>"
2498   [
2499    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2500         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2501                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2502          VMINNMAQ_F))
2503   ]
2504   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2505   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2506   [(set_attr "type" "mve_move")
2507 ])
2508
2509 ;;
2510 ;; [vminnmavq_f])
2511 ;;
2512 (define_insn "mve_vminnmavq_f<mode>"
2513   [
2514    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2515         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2516                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2517          VMINNMAVQ_F))
2518   ]
2519   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2520   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2521   [(set_attr "type" "mve_move")
2522 ])
2523
2524 ;;
2525 ;; [vminnmq_f])
2526 ;;
2527 (define_insn "mve_vminnmq_f<mode>"
2528   [
2529    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2530         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2531                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2532   ]
2533   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2534   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2535   [(set_attr "type" "mve_move")
2536 ])
2537
2538 ;;
2539 ;; [vminnmvq_f])
2540 ;;
2541 (define_insn "mve_vminnmvq_f<mode>"
2542   [
2543    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2544         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2545                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2546          VMINNMVQ_F))
2547   ]
2548   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2549   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2550   [(set_attr "type" "mve_move")
2551 ])
2552
2553 ;;
2554 ;; [vmlaldavq_u, vmlaldavq_s])
2555 ;;
2556 (define_insn "mve_vmlaldavq_<supf><mode>"
2557   [
2558    (set (match_operand:DI 0 "s_register_operand" "=r")
2559         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2560                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2561          VMLALDAVQ))
2562   ]
2563   "TARGET_HAVE_MVE"
2564   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2565   [(set_attr "type" "mve_move")
2566 ])
2567
2568 ;;
2569 ;; [vmlaldavxq_s])
2570 ;;
2571 (define_insn "mve_vmlaldavxq_s<mode>"
2572   [
2573    (set (match_operand:DI 0 "s_register_operand" "=r")
2574         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2575                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2576          VMLALDAVXQ_S))
2577   ]
2578   "TARGET_HAVE_MVE"
2579   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2580   [(set_attr "type" "mve_move")
2581 ])
2582
2583 ;;
2584 ;; [vmlsldavq_s])
2585 ;;
2586 (define_insn "mve_vmlsldavq_s<mode>"
2587   [
2588    (set (match_operand:DI 0 "s_register_operand" "=r")
2589         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2590                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2591          VMLSLDAVQ_S))
2592   ]
2593   "TARGET_HAVE_MVE"
2594   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2595   [(set_attr "type" "mve_move")
2596 ])
2597
2598 ;;
2599 ;; [vmlsldavxq_s])
2600 ;;
2601 (define_insn "mve_vmlsldavxq_s<mode>"
2602   [
2603    (set (match_operand:DI 0 "s_register_operand" "=r")
2604         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2605                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2606          VMLSLDAVXQ_S))
2607   ]
2608   "TARGET_HAVE_MVE"
2609   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2610   [(set_attr "type" "mve_move")
2611 ])
2612
2613 ;;
2614 ;; [vmovnbq_u, vmovnbq_s])
2615 ;;
2616 (define_insn "mve_vmovnbq_<supf><mode>"
2617   [
2618    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2619         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2620                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2621          VMOVNBQ))
2622   ]
2623   "TARGET_HAVE_MVE"
2624   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2625   [(set_attr "type" "mve_move")
2626 ])
2627
2628 ;;
2629 ;; [vmovntq_s, vmovntq_u])
2630 ;;
2631 (define_insn "mve_vmovntq_<supf><mode>"
2632   [
2633    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2634         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2635                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2636          VMOVNTQ))
2637   ]
2638   "TARGET_HAVE_MVE"
2639   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2640   [(set_attr "type" "mve_move")
2641 ])
2642
2643 ;;
2644 ;; [vmulq_f])
2645 ;;
2646 (define_insn "mve_vmulq_f<mode>"
2647   [
2648    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2649         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2650                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2651   ]
2652   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2653   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2654   [(set_attr "type" "mve_move")
2655 ])
2656
2657 ;;
2658 ;; [vmulq_n_f])
2659 ;;
2660 (define_insn "mve_vmulq_n_f<mode>"
2661   [
2662    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2663         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2664                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2665          VMULQ_N_F))
2666   ]
2667   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2668   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2669   [(set_attr "type" "mve_move")
2670 ])
2671
2672 ;;
2673 ;; [vornq_f])
2674 ;;
2675 (define_insn "mve_vornq_f<mode>"
2676   [
2677    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2678         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2679                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2680          VORNQ_F))
2681   ]
2682   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2683   "vorn %q0, %q1, %q2"
2684   [(set_attr "type" "mve_move")
2685 ])
2686
2687 ;;
2688 ;; [vorrq_f])
2689 ;;
2690 (define_insn "mve_vorrq_f<mode>"
2691   [
2692    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2693         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2694                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2695   ]
2696   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2697   "vorr %q0, %q1, %q2"
2698   [(set_attr "type" "mve_move")
2699 ])
2700
2701 ;;
2702 ;; [vorrq_n_u, vorrq_n_s])
2703 ;;
2704 (define_insn "mve_vorrq_n_<supf><mode>"
2705   [
2706    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2707         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2708                        (match_operand:SI 2 "immediate_operand" "i")]
2709          VORRQ_N))
2710   ]
2711   "TARGET_HAVE_MVE"
2712   "vorr.i%#<V_sz_elem>  %q0, %2"
2713   [(set_attr "type" "mve_move")
2714 ])
2715
2716 ;;
2717 ;; [vqdmullbq_n_s])
2718 ;;
2719 (define_insn "mve_vqdmullbq_n_s<mode>"
2720   [
2721    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2722         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2723                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2724          VQDMULLBQ_N_S))
2725   ]
2726   "TARGET_HAVE_MVE"
2727   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2728   [(set_attr "type" "mve_move")
2729 ])
2730
2731 ;;
2732 ;; [vqdmullbq_s])
2733 ;;
2734 (define_insn "mve_vqdmullbq_s<mode>"
2735   [
2736    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2737         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2738                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2739          VQDMULLBQ_S))
2740   ]
2741   "TARGET_HAVE_MVE"
2742   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2743   [(set_attr "type" "mve_move")
2744 ])
2745
2746 ;;
2747 ;; [vqdmulltq_n_s])
2748 ;;
2749 (define_insn "mve_vqdmulltq_n_s<mode>"
2750   [
2751    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2752         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2753                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2754          VQDMULLTQ_N_S))
2755   ]
2756   "TARGET_HAVE_MVE"
2757   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2758   [(set_attr "type" "mve_move")
2759 ])
2760
2761 ;;
2762 ;; [vqdmulltq_s])
2763 ;;
2764 (define_insn "mve_vqdmulltq_s<mode>"
2765   [
2766    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2767         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2768                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2769          VQDMULLTQ_S))
2770   ]
2771   "TARGET_HAVE_MVE"
2772   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2773   [(set_attr "type" "mve_move")
2774 ])
2775
2776 ;;
2777 ;; [vqmovnbq_u, vqmovnbq_s])
2778 ;;
2779 (define_insn "mve_vqmovnbq_<supf><mode>"
2780   [
2781    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2782         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2783                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2784          VQMOVNBQ))
2785   ]
2786   "TARGET_HAVE_MVE"
2787   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2788   [(set_attr "type" "mve_move")
2789 ])
2790
2791 ;;
2792 ;; [vqmovntq_u, vqmovntq_s])
2793 ;;
2794 (define_insn "mve_vqmovntq_<supf><mode>"
2795   [
2796    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2797         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2798                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2799          VQMOVNTQ))
2800   ]
2801   "TARGET_HAVE_MVE"
2802   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2803   [(set_attr "type" "mve_move")
2804 ])
2805
2806 ;;
2807 ;; [vqmovunbq_s])
2808 ;;
2809 (define_insn "mve_vqmovunbq_s<mode>"
2810   [
2811    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2812         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2813                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2814          VQMOVUNBQ_S))
2815   ]
2816   "TARGET_HAVE_MVE"
2817   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2818   [(set_attr "type" "mve_move")
2819 ])
2820
2821 ;;
2822 ;; [vqmovuntq_s])
2823 ;;
2824 (define_insn "mve_vqmovuntq_s<mode>"
2825   [
2826    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2827         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2828                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2829          VQMOVUNTQ_S))
2830   ]
2831   "TARGET_HAVE_MVE"
2832   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2833   [(set_attr "type" "mve_move")
2834 ])
2835
2836 ;;
2837 ;; [vrmlaldavhxq_s])
2838 ;;
2839 (define_insn "mve_vrmlaldavhxq_sv4si"
2840   [
2841    (set (match_operand:DI 0 "s_register_operand" "=r")
2842         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2843                     (match_operand:V4SI 2 "s_register_operand" "w")]
2844          VRMLALDAVHXQ_S))
2845   ]
2846   "TARGET_HAVE_MVE"
2847   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2848   [(set_attr "type" "mve_move")
2849 ])
2850
2851 ;;
2852 ;; [vrmlsldavhq_s])
2853 ;;
2854 (define_insn "mve_vrmlsldavhq_sv4si"
2855   [
2856    (set (match_operand:DI 0 "s_register_operand" "=r")
2857         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2858                     (match_operand:V4SI 2 "s_register_operand" "w")]
2859          VRMLSLDAVHQ_S))
2860   ]
2861   "TARGET_HAVE_MVE"
2862   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2863   [(set_attr "type" "mve_move")
2864 ])
2865
2866 ;;
2867 ;; [vrmlsldavhxq_s])
2868 ;;
2869 (define_insn "mve_vrmlsldavhxq_sv4si"
2870   [
2871    (set (match_operand:DI 0 "s_register_operand" "=r")
2872         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2873                     (match_operand:V4SI 2 "s_register_operand" "w")]
2874          VRMLSLDAVHXQ_S))
2875   ]
2876   "TARGET_HAVE_MVE"
2877   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2878   [(set_attr "type" "mve_move")
2879 ])
2880
2881 ;;
2882 ;; [vshllbq_n_s, vshllbq_n_u])
2883 ;;
2884 (define_insn "mve_vshllbq_n_<supf><mode>"
2885   [
2886    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2887         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2888                                   (match_operand:SI 2 "immediate_operand" "i")]
2889          VSHLLBQ_N))
2890   ]
2891   "TARGET_HAVE_MVE"
2892   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2893   [(set_attr "type" "mve_move")
2894 ])
2895
2896 ;;
2897 ;; [vshlltq_n_u, vshlltq_n_s])
2898 ;;
2899 (define_insn "mve_vshlltq_n_<supf><mode>"
2900   [
2901    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2902         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2903                                   (match_operand:SI 2 "immediate_operand" "i")]
2904          VSHLLTQ_N))
2905   ]
2906   "TARGET_HAVE_MVE"
2907   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2908   [(set_attr "type" "mve_move")
2909 ])
2910
2911 ;;
2912 ;; [vsubq_f])
2913 ;;
2914 (define_insn "mve_vsubq_f<mode>"
2915   [
2916    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2917         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2918                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2919   ]
2920   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2921   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2922   [(set_attr "type" "mve_move")
2923 ])
2924
2925 ;;
2926 ;; [vmulltq_poly_p])
2927 ;;
2928 (define_insn "mve_vmulltq_poly_p<mode>"
2929   [
2930    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2931         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2932                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2933          VMULLTQ_POLY_P))
2934   ]
2935   "TARGET_HAVE_MVE"
2936   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2937   [(set_attr "type" "mve_move")
2938 ])
2939
2940 ;;
2941 ;; [vmullbq_poly_p])
2942 ;;
2943 (define_insn "mve_vmullbq_poly_p<mode>"
2944   [
2945    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2946         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2947                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2948          VMULLBQ_POLY_P))
2949   ]
2950   "TARGET_HAVE_MVE"
2951   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2952   [(set_attr "type" "mve_move")
2953 ])
2954
2955 ;;
2956 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2957 ;;
2958 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2959   [
2960    (set (match_operand:DI 0 "s_register_operand" "=r")
2961         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2962                     (match_operand:V4SI 2 "s_register_operand" "w")]
2963          VRMLALDAVHQ))
2964   ]
2965   "TARGET_HAVE_MVE"
2966   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2967   [(set_attr "type" "mve_move")
2968 ])
2969
2970 ;;
2971 ;; [vbicq_m_n_s, vbicq_m_n_u])
2972 ;;
2973 (define_insn "mve_vbicq_m_n_<supf><mode>"
2974   [
2975    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2976         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2977                        (match_operand:SI 2 "immediate_operand" "i")
2978                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2979          VBICQ_M_N))
2980   ]
2981   "TARGET_HAVE_MVE"
2982   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2983   [(set_attr "type" "mve_move")
2984    (set_attr "length""8")])
2985 ;;
2986 ;; [vcmpeqq_m_f])
2987 ;;
2988 (define_insn "mve_vcmpeqq_m_f<mode>"
2989   [
2990    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2991         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2992                     (match_operand:MVE_0 2 "s_register_operand" "w")
2993                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2994          VCMPEQQ_M_F))
2995   ]
2996   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2997   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2998   [(set_attr "type" "mve_move")
2999    (set_attr "length""8")])
3000 ;;
3001 ;; [vcvtaq_m_u, vcvtaq_m_s])
3002 ;;
3003 (define_insn "mve_vcvtaq_m_<supf><mode>"
3004   [
3005    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3006         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3007                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3008                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3009          VCVTAQ_M))
3010   ]
3011   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3012   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3013   [(set_attr "type" "mve_move")
3014    (set_attr "length""8")])
3015 ;;
3016 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3017 ;;
3018 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3019   [
3020    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3021         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3022                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3023                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3024          VCVTQ_M_TO_F))
3025   ]
3026   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3027   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3028   [(set_attr "type" "mve_move")
3029    (set_attr "length""8")])
3030 ;;
3031 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3032 ;;
3033 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3034   [
3035    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3036         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3037                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3038                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3039          VQRSHRNBQ_N))
3040   ]
3041   "TARGET_HAVE_MVE"
3042   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3043   [(set_attr "type" "mve_move")
3044 ])
3045 ;;
3046 ;; [vqrshrunbq_n_s])
3047 ;;
3048 (define_insn "mve_vqrshrunbq_n_s<mode>"
3049   [
3050    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3051         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3052                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3053                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3054          VQRSHRUNBQ_N_S))
3055   ]
3056   "TARGET_HAVE_MVE"
3057   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3058   [(set_attr "type" "mve_move")
3059 ])
3060 ;;
3061 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3062 ;;
3063 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3064   [
3065    (set (match_operand:DI 0 "s_register_operand" "=r")
3066         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3067                     (match_operand:V4SI 2 "s_register_operand" "w")
3068                     (match_operand:V4SI 3 "s_register_operand" "w")]
3069          VRMLALDAVHAQ))
3070   ]
3071   "TARGET_HAVE_MVE"
3072   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3073   [(set_attr "type" "mve_move")
3074 ])
3075
3076 ;;
3077 ;; [vabavq_s, vabavq_u])
3078 ;;
3079 (define_insn "mve_vabavq_<supf><mode>"
3080   [
3081    (set (match_operand:SI 0 "s_register_operand" "=r")
3082         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3083                     (match_operand:MVE_2 2 "s_register_operand" "w")
3084                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3085          VABAVQ))
3086   ]
3087   "TARGET_HAVE_MVE"
3088   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3089   [(set_attr "type" "mve_move")
3090 ])
3091
3092 ;;
3093 ;; [vshlcq_u vshlcq_s]
3094 ;;
3095 (define_expand "mve_vshlcq_vec_<supf><mode>"
3096  [(match_operand:MVE_2 0 "s_register_operand")
3097   (match_operand:MVE_2 1 "s_register_operand")
3098   (match_operand:SI 2 "s_register_operand")
3099   (match_operand:SI 3 "mve_imm_32")
3100   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3101  "TARGET_HAVE_MVE"
3102 {
3103   rtx ignore_wb = gen_reg_rtx (SImode);
3104   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3105                                       operands[2], operands[3]));
3106   DONE;
3107 })
3108
3109 (define_expand "mve_vshlcq_carry_<supf><mode>"
3110  [(match_operand:SI 0 "s_register_operand")
3111   (match_operand:MVE_2 1 "s_register_operand")
3112   (match_operand:SI 2 "s_register_operand")
3113   (match_operand:SI 3 "mve_imm_32")
3114   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3115  "TARGET_HAVE_MVE"
3116 {
3117   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3118   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3119                                       operands[2], operands[3]));
3120   DONE;
3121 })
3122
3123 (define_insn "mve_vshlcq_<supf><mode>"
3124  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3125        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3126                       (match_operand:SI 3 "s_register_operand" "1")
3127                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3128         VSHLCQ))
3129   (set (match_operand:SI  1 "s_register_operand" "=r")
3130        (unspec:SI [(match_dup 2)
3131                    (match_dup 3)
3132                    (match_dup 4)]
3133         VSHLCQ))]
3134  "TARGET_HAVE_MVE"
3135  "vshlc %q0, %1, %4")
3136
3137 ;;
3138 ;; [vabsq_m_s])
3139 ;;
3140 (define_insn "mve_vabsq_m_s<mode>"
3141   [
3142    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3143         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3144                        (match_operand:MVE_2 2 "s_register_operand" "w")
3145                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3146          VABSQ_M_S))
3147   ]
3148   "TARGET_HAVE_MVE"
3149   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3150   [(set_attr "type" "mve_move")
3151    (set_attr "length""8")])
3152
3153 ;;
3154 ;; [vaddvaq_p_u, vaddvaq_p_s])
3155 ;;
3156 (define_insn "mve_vaddvaq_p_<supf><mode>"
3157   [
3158    (set (match_operand:SI 0 "s_register_operand" "=Te")
3159         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3160                        (match_operand:MVE_2 2 "s_register_operand" "w")
3161                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3162          VADDVAQ_P))
3163   ]
3164   "TARGET_HAVE_MVE"
3165   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3166   [(set_attr "type" "mve_move")
3167    (set_attr "length""8")])
3168
3169 ;;
3170 ;; [vclsq_m_s])
3171 ;;
3172 (define_insn "mve_vclsq_m_s<mode>"
3173   [
3174    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3175         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3176                        (match_operand:MVE_2 2 "s_register_operand" "w")
3177                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3178          VCLSQ_M_S))
3179   ]
3180   "TARGET_HAVE_MVE"
3181   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3182   [(set_attr "type" "mve_move")
3183    (set_attr "length""8")])
3184
3185 ;;
3186 ;; [vclzq_m_s, vclzq_m_u])
3187 ;;
3188 (define_insn "mve_vclzq_m_<supf><mode>"
3189   [
3190    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3191         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3192                        (match_operand:MVE_2 2 "s_register_operand" "w")
3193                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3194          VCLZQ_M))
3195   ]
3196   "TARGET_HAVE_MVE"
3197   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3198   [(set_attr "type" "mve_move")
3199    (set_attr "length""8")])
3200
3201 ;;
3202 ;; [vcmpcsq_m_n_u])
3203 ;;
3204 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3205   [
3206    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3207         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3208                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3209                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3210          VCMPCSQ_M_N_U))
3211   ]
3212   "TARGET_HAVE_MVE"
3213   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3214   [(set_attr "type" "mve_move")
3215    (set_attr "length""8")])
3216
3217 ;;
3218 ;; [vcmpcsq_m_u])
3219 ;;
3220 (define_insn "mve_vcmpcsq_m_u<mode>"
3221   [
3222    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3223         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3224                        (match_operand:MVE_2 2 "s_register_operand" "w")
3225                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3226          VCMPCSQ_M_U))
3227   ]
3228   "TARGET_HAVE_MVE"
3229   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3230   [(set_attr "type" "mve_move")
3231    (set_attr "length""8")])
3232
3233 ;;
3234 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3235 ;;
3236 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3237   [
3238    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3239         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3240                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3241                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3242          VCMPEQQ_M_N))
3243   ]
3244   "TARGET_HAVE_MVE"
3245   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3246   [(set_attr "type" "mve_move")
3247    (set_attr "length""8")])
3248
3249 ;;
3250 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3251 ;;
3252 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3253   [
3254    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3255         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3256                        (match_operand:MVE_2 2 "s_register_operand" "w")
3257                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3258          VCMPEQQ_M))
3259   ]
3260   "TARGET_HAVE_MVE"
3261   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3262   [(set_attr "type" "mve_move")
3263    (set_attr "length""8")])
3264
3265 ;;
3266 ;; [vcmpgeq_m_n_s])
3267 ;;
3268 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3269   [
3270    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3271         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3272                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3273                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3274          VCMPGEQ_M_N_S))
3275   ]
3276   "TARGET_HAVE_MVE"
3277   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3278   [(set_attr "type" "mve_move")
3279    (set_attr "length""8")])
3280
3281 ;;
3282 ;; [vcmpgeq_m_s])
3283 ;;
3284 (define_insn "mve_vcmpgeq_m_s<mode>"
3285   [
3286    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3287         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3288                        (match_operand:MVE_2 2 "s_register_operand" "w")
3289                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3290          VCMPGEQ_M_S))
3291   ]
3292   "TARGET_HAVE_MVE"
3293   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3294   [(set_attr "type" "mve_move")
3295    (set_attr "length""8")])
3296
3297 ;;
3298 ;; [vcmpgtq_m_n_s])
3299 ;;
3300 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3301   [
3302    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3303         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3304                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3305                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3306          VCMPGTQ_M_N_S))
3307   ]
3308   "TARGET_HAVE_MVE"
3309   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3310   [(set_attr "type" "mve_move")
3311    (set_attr "length""8")])
3312
3313 ;;
3314 ;; [vcmpgtq_m_s])
3315 ;;
3316 (define_insn "mve_vcmpgtq_m_s<mode>"
3317   [
3318    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3319         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3320                        (match_operand:MVE_2 2 "s_register_operand" "w")
3321                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3322          VCMPGTQ_M_S))
3323   ]
3324   "TARGET_HAVE_MVE"
3325   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3326   [(set_attr "type" "mve_move")
3327    (set_attr "length""8")])
3328
3329 ;;
3330 ;; [vcmphiq_m_n_u])
3331 ;;
3332 (define_insn "mve_vcmphiq_m_n_u<mode>"
3333   [
3334    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3335         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3336                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3337                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3338          VCMPHIQ_M_N_U))
3339   ]
3340   "TARGET_HAVE_MVE"
3341   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3342   [(set_attr "type" "mve_move")
3343    (set_attr "length""8")])
3344
3345 ;;
3346 ;; [vcmphiq_m_u])
3347 ;;
3348 (define_insn "mve_vcmphiq_m_u<mode>"
3349   [
3350    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3351         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3352                        (match_operand:MVE_2 2 "s_register_operand" "w")
3353                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3354          VCMPHIQ_M_U))
3355   ]
3356   "TARGET_HAVE_MVE"
3357   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3358   [(set_attr "type" "mve_move")
3359    (set_attr "length""8")])
3360
3361 ;;
3362 ;; [vcmpleq_m_n_s])
3363 ;;
3364 (define_insn "mve_vcmpleq_m_n_s<mode>"
3365   [
3366    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3367         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3368                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3369                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3370          VCMPLEQ_M_N_S))
3371   ]
3372   "TARGET_HAVE_MVE"
3373   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3374   [(set_attr "type" "mve_move")
3375    (set_attr "length""8")])
3376
3377 ;;
3378 ;; [vcmpleq_m_s])
3379 ;;
3380 (define_insn "mve_vcmpleq_m_s<mode>"
3381   [
3382    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3383         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3384                        (match_operand:MVE_2 2 "s_register_operand" "w")
3385                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3386          VCMPLEQ_M_S))
3387   ]
3388   "TARGET_HAVE_MVE"
3389   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3390   [(set_attr "type" "mve_move")
3391    (set_attr "length""8")])
3392
3393 ;;
3394 ;; [vcmpltq_m_n_s])
3395 ;;
3396 (define_insn "mve_vcmpltq_m_n_s<mode>"
3397   [
3398    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3399         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3400                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3401                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3402          VCMPLTQ_M_N_S))
3403   ]
3404   "TARGET_HAVE_MVE"
3405   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3406   [(set_attr "type" "mve_move")
3407    (set_attr "length""8")])
3408
3409 ;;
3410 ;; [vcmpltq_m_s])
3411 ;;
3412 (define_insn "mve_vcmpltq_m_s<mode>"
3413   [
3414    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3415         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3416                        (match_operand:MVE_2 2 "s_register_operand" "w")
3417                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3418          VCMPLTQ_M_S))
3419   ]
3420   "TARGET_HAVE_MVE"
3421   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3422   [(set_attr "type" "mve_move")
3423    (set_attr "length""8")])
3424
3425 ;;
3426 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3427 ;;
3428 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3429   [
3430    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3431         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3432                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3433                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3434          VCMPNEQ_M_N))
3435   ]
3436   "TARGET_HAVE_MVE"
3437   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3438   [(set_attr "type" "mve_move")
3439    (set_attr "length""8")])
3440
3441 ;;
3442 ;; [vcmpneq_m_s, vcmpneq_m_u])
3443 ;;
3444 (define_insn "mve_vcmpneq_m_<supf><mode>"
3445   [
3446    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3447         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3448                        (match_operand:MVE_2 2 "s_register_operand" "w")
3449                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3450          VCMPNEQ_M))
3451   ]
3452   "TARGET_HAVE_MVE"
3453   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3454   [(set_attr "type" "mve_move")
3455    (set_attr "length""8")])
3456
3457 ;;
3458 ;; [vdupq_m_n_s, vdupq_m_n_u])
3459 ;;
3460 (define_insn "mve_vdupq_m_n_<supf><mode>"
3461   [
3462    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3463         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3464                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3465                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3466          VDUPQ_M_N))
3467   ]
3468   "TARGET_HAVE_MVE"
3469   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3470   [(set_attr "type" "mve_move")
3471    (set_attr "length""8")])
3472
3473 ;;
3474 ;; [vmaxaq_m_s])
3475 ;;
3476 (define_insn "mve_vmaxaq_m_s<mode>"
3477   [
3478    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3479         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3480                        (match_operand:MVE_2 2 "s_register_operand" "w")
3481                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3482          VMAXAQ_M_S))
3483   ]
3484   "TARGET_HAVE_MVE"
3485   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3486   [(set_attr "type" "mve_move")
3487    (set_attr "length""8")])
3488
3489 ;;
3490 ;; [vmaxavq_p_s])
3491 ;;
3492 (define_insn "mve_vmaxavq_p_s<mode>"
3493   [
3494    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3495         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3496                        (match_operand:MVE_2 2 "s_register_operand" "w")
3497                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3498          VMAXAVQ_P_S))
3499   ]
3500   "TARGET_HAVE_MVE"
3501   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3502   [(set_attr "type" "mve_move")
3503    (set_attr "length""8")])
3504
3505 ;;
3506 ;; [vmaxvq_p_u, vmaxvq_p_s])
3507 ;;
3508 (define_insn "mve_vmaxvq_p_<supf><mode>"
3509   [
3510    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3511         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3512                        (match_operand:MVE_2 2 "s_register_operand" "w")
3513                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3514          VMAXVQ_P))
3515   ]
3516   "TARGET_HAVE_MVE"
3517   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3518   [(set_attr "type" "mve_move")
3519    (set_attr "length""8")])
3520
3521 ;;
3522 ;; [vminaq_m_s])
3523 ;;
3524 (define_insn "mve_vminaq_m_s<mode>"
3525   [
3526    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3527         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3528                        (match_operand:MVE_2 2 "s_register_operand" "w")
3529                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3530          VMINAQ_M_S))
3531   ]
3532   "TARGET_HAVE_MVE"
3533   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3534   [(set_attr "type" "mve_move")
3535    (set_attr "length""8")])
3536
3537 ;;
3538 ;; [vminavq_p_s])
3539 ;;
3540 (define_insn "mve_vminavq_p_s<mode>"
3541   [
3542    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3543         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3544                        (match_operand:MVE_2 2 "s_register_operand" "w")
3545                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3546          VMINAVQ_P_S))
3547   ]
3548   "TARGET_HAVE_MVE"
3549   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3550   [(set_attr "type" "mve_move")
3551    (set_attr "length""8")])
3552
3553 ;;
3554 ;; [vminvq_p_s, vminvq_p_u])
3555 ;;
3556 (define_insn "mve_vminvq_p_<supf><mode>"
3557   [
3558    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3559         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3560                        (match_operand:MVE_2 2 "s_register_operand" "w")
3561                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3562          VMINVQ_P))
3563   ]
3564   "TARGET_HAVE_MVE"
3565   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3566   [(set_attr "type" "mve_move")
3567    (set_attr "length""8")])
3568
3569 ;;
3570 ;; [vmladavaq_u, vmladavaq_s])
3571 ;;
3572 (define_insn "mve_vmladavaq_<supf><mode>"
3573   [
3574    (set (match_operand:SI 0 "s_register_operand" "=Te")
3575         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3576                        (match_operand:MVE_2 2 "s_register_operand" "w")
3577                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3578          VMLADAVAQ))
3579   ]
3580   "TARGET_HAVE_MVE"
3581   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3582   [(set_attr "type" "mve_move")
3583 ])
3584
3585 ;;
3586 ;; [vmladavq_p_u, vmladavq_p_s])
3587 ;;
3588 (define_insn "mve_vmladavq_p_<supf><mode>"
3589   [
3590    (set (match_operand:SI 0 "s_register_operand" "=Te")
3591         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3592                        (match_operand:MVE_2 2 "s_register_operand" "w")
3593                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3594          VMLADAVQ_P))
3595   ]
3596   "TARGET_HAVE_MVE"
3597   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3598   [(set_attr "type" "mve_move")
3599    (set_attr "length""8")])
3600
3601 ;;
3602 ;; [vmladavxq_p_s])
3603 ;;
3604 (define_insn "mve_vmladavxq_p_s<mode>"
3605   [
3606    (set (match_operand:SI 0 "s_register_operand" "=Te")
3607         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3608                        (match_operand:MVE_2 2 "s_register_operand" "w")
3609                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3610          VMLADAVXQ_P_S))
3611   ]
3612   "TARGET_HAVE_MVE"
3613   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3614   [(set_attr "type" "mve_move")
3615    (set_attr "length""8")])
3616
3617 ;;
3618 ;; [vmlaq_n_u, vmlaq_n_s])
3619 ;;
3620 (define_insn "mve_vmlaq_n_<supf><mode>"
3621   [
3622    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3623         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3624                        (match_operand:MVE_2 2 "s_register_operand" "w")
3625                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3626          VMLAQ_N))
3627   ]
3628   "TARGET_HAVE_MVE"
3629   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3630   [(set_attr "type" "mve_move")
3631 ])
3632
3633 ;;
3634 ;; [vmlasq_n_u, vmlasq_n_s])
3635 ;;
3636 (define_insn "mve_vmlasq_n_<supf><mode>"
3637   [
3638    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3639         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3640                        (match_operand:MVE_2 2 "s_register_operand" "w")
3641                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3642          VMLASQ_N))
3643   ]
3644   "TARGET_HAVE_MVE"
3645   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3646   [(set_attr "type" "mve_move")
3647 ])
3648
3649 ;;
3650 ;; [vmlsdavq_p_s])
3651 ;;
3652 (define_insn "mve_vmlsdavq_p_s<mode>"
3653   [
3654    (set (match_operand:SI 0 "s_register_operand" "=Te")
3655         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3656                        (match_operand:MVE_2 2 "s_register_operand" "w")
3657                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3658          VMLSDAVQ_P_S))
3659   ]
3660   "TARGET_HAVE_MVE"
3661   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3662   [(set_attr "type" "mve_move")
3663    (set_attr "length""8")])
3664
3665 ;;
3666 ;; [vmlsdavxq_p_s])
3667 ;;
3668 (define_insn "mve_vmlsdavxq_p_s<mode>"
3669   [
3670    (set (match_operand:SI 0 "s_register_operand" "=Te")
3671         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3672                        (match_operand:MVE_2 2 "s_register_operand" "w")
3673                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3674          VMLSDAVXQ_P_S))
3675   ]
3676   "TARGET_HAVE_MVE"
3677   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3678   [(set_attr "type" "mve_move")
3679    (set_attr "length""8")])
3680
3681 ;;
3682 ;; [vmvnq_m_s, vmvnq_m_u])
3683 ;;
3684 (define_insn "mve_vmvnq_m_<supf><mode>"
3685   [
3686    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3687         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3688                        (match_operand:MVE_2 2 "s_register_operand" "w")
3689                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3690          VMVNQ_M))
3691   ]
3692   "TARGET_HAVE_MVE"
3693   "vpst\;vmvnt %q0, %q2"
3694   [(set_attr "type" "mve_move")
3695    (set_attr "length""8")])
3696
3697 ;;
3698 ;; [vnegq_m_s])
3699 ;;
3700 (define_insn "mve_vnegq_m_s<mode>"
3701   [
3702    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3703         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3704                        (match_operand:MVE_2 2 "s_register_operand" "w")
3705                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3706          VNEGQ_M_S))
3707   ]
3708   "TARGET_HAVE_MVE"
3709   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3710   [(set_attr "type" "mve_move")
3711    (set_attr "length""8")])
3712
3713 ;;
3714 ;; [vpselq_u, vpselq_s])
3715 ;;
3716 (define_insn "mve_vpselq_<supf><mode>"
3717   [
3718    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3719         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3720                        (match_operand:MVE_1 2 "s_register_operand" "w")
3721                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3722          VPSELQ))
3723   ]
3724   "TARGET_HAVE_MVE"
3725   "vpsel %q0, %q1, %q2"
3726   [(set_attr "type" "mve_move")
3727 ])
3728
3729 ;;
3730 ;; [vqabsq_m_s])
3731 ;;
3732 (define_insn "mve_vqabsq_m_s<mode>"
3733   [
3734    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3735         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3736                        (match_operand:MVE_2 2 "s_register_operand" "w")
3737                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3738          VQABSQ_M_S))
3739   ]
3740   "TARGET_HAVE_MVE"
3741   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3742   [(set_attr "type" "mve_move")
3743    (set_attr "length""8")])
3744
3745 ;;
3746 ;; [vqdmlahq_n_s])
3747 ;;
3748 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3749   [
3750    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3751         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3752                        (match_operand:MVE_2 2 "s_register_operand" "w")
3753                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3754          VQDMLAHQ_N))
3755   ]
3756   "TARGET_HAVE_MVE"
3757   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3758   [(set_attr "type" "mve_move")
3759 ])
3760
3761 ;;
3762 ;; [vqdmlashq_n_s])
3763 ;;
3764 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3765   [
3766    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3767         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3768                        (match_operand:MVE_2 2 "s_register_operand" "w")
3769                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3770          VQDMLASHQ_N))
3771   ]
3772   "TARGET_HAVE_MVE"
3773   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3774   [(set_attr "type" "mve_move")
3775 ])
3776
3777 ;;
3778 ;; [vqnegq_m_s])
3779 ;;
3780 (define_insn "mve_vqnegq_m_s<mode>"
3781   [
3782    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3783         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3784                        (match_operand:MVE_2 2 "s_register_operand" "w")
3785                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3786          VQNEGQ_M_S))
3787   ]
3788   "TARGET_HAVE_MVE"
3789   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3790   [(set_attr "type" "mve_move")
3791    (set_attr "length""8")])
3792
3793 ;;
3794 ;; [vqrdmladhq_s])
3795 ;;
3796 (define_insn "mve_vqrdmladhq_s<mode>"
3797   [
3798    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3799         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3800                        (match_operand:MVE_2 2 "s_register_operand" "w")
3801                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3802          VQRDMLADHQ_S))
3803   ]
3804   "TARGET_HAVE_MVE"
3805   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3806   [(set_attr "type" "mve_move")
3807 ])
3808
3809 ;;
3810 ;; [vqrdmladhxq_s])
3811 ;;
3812 (define_insn "mve_vqrdmladhxq_s<mode>"
3813   [
3814    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3815         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3816                        (match_operand:MVE_2 2 "s_register_operand" "w")
3817                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3818          VQRDMLADHXQ_S))
3819   ]
3820   "TARGET_HAVE_MVE"
3821   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3822   [(set_attr "type" "mve_move")
3823 ])
3824
3825 ;;
3826 ;; [vqrdmlahq_n_s])
3827 ;;
3828 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3829   [
3830    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3831         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3832                        (match_operand:MVE_2 2 "s_register_operand" "w")
3833                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3834          VQRDMLAHQ_N))
3835   ]
3836   "TARGET_HAVE_MVE"
3837   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3838   [(set_attr "type" "mve_move")
3839 ])
3840
3841 ;;
3842 ;; [vqrdmlashq_n_s])
3843 ;;
3844 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3845   [
3846    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3847         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3848                        (match_operand:MVE_2 2 "s_register_operand" "w")
3849                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3850          VQRDMLASHQ_N))
3851   ]
3852   "TARGET_HAVE_MVE"
3853   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3854   [(set_attr "type" "mve_move")
3855 ])
3856
3857 ;;
3858 ;; [vqrdmlsdhq_s])
3859 ;;
3860 (define_insn "mve_vqrdmlsdhq_s<mode>"
3861   [
3862    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3863         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3864                        (match_operand:MVE_2 2 "s_register_operand" "w")
3865                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3866          VQRDMLSDHQ_S))
3867   ]
3868   "TARGET_HAVE_MVE"
3869   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3870   [(set_attr "type" "mve_move")
3871 ])
3872
3873 ;;
3874 ;; [vqrdmlsdhxq_s])
3875 ;;
3876 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3877   [
3878    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3879         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3880                        (match_operand:MVE_2 2 "s_register_operand" "w")
3881                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3882          VQRDMLSDHXQ_S))
3883   ]
3884   "TARGET_HAVE_MVE"
3885   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3886   [(set_attr "type" "mve_move")
3887 ])
3888
3889 ;;
3890 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3891 ;;
3892 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3893   [
3894    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3895         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3896                        (match_operand:SI 2 "s_register_operand" "r")
3897                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3898          VQRSHLQ_M_N))
3899   ]
3900   "TARGET_HAVE_MVE"
3901   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3902   [(set_attr "type" "mve_move")
3903    (set_attr "length""8")])
3904
3905 ;;
3906 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3907 ;;
3908 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3909   [
3910    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3911         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3912                        (match_operand:SI 2 "s_register_operand" "r")
3913                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3914          VQSHLQ_M_R))
3915   ]
3916   "TARGET_HAVE_MVE"
3917   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3918   [(set_attr "type" "mve_move")
3919    (set_attr "length""8")])
3920
3921 ;;
3922 ;; [vrev64q_m_u, vrev64q_m_s])
3923 ;;
3924 (define_insn "mve_vrev64q_m_<supf><mode>"
3925   [
3926    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3927         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3928                        (match_operand:MVE_2 2 "s_register_operand" "w")
3929                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3930          VREV64Q_M))
3931   ]
3932   "TARGET_HAVE_MVE"
3933   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3934   [(set_attr "type" "mve_move")
3935    (set_attr "length""8")])
3936
3937 ;;
3938 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3939 ;;
3940 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3941   [
3942    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3943         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3944                        (match_operand:SI 2 "s_register_operand" "r")
3945                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3946          VRSHLQ_M_N))
3947   ]
3948   "TARGET_HAVE_MVE"
3949   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3950   [(set_attr "type" "mve_move")
3951    (set_attr "length""8")])
3952
3953 ;;
3954 ;; [vshlq_m_r_u, vshlq_m_r_s])
3955 ;;
3956 (define_insn "mve_vshlq_m_r_<supf><mode>"
3957   [
3958    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3959         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3960                        (match_operand:SI 2 "s_register_operand" "r")
3961                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3962          VSHLQ_M_R))
3963   ]
3964   "TARGET_HAVE_MVE"
3965   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3966   [(set_attr "type" "mve_move")
3967    (set_attr "length""8")])
3968
3969 ;;
3970 ;; [vsliq_n_u, vsliq_n_s])
3971 ;;
3972 (define_insn "mve_vsliq_n_<supf><mode>"
3973   [
3974    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3975         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3976                        (match_operand:MVE_2 2 "s_register_operand" "w")
3977                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3978          VSLIQ_N))
3979   ]
3980   "TARGET_HAVE_MVE"
3981   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3982   [(set_attr "type" "mve_move")
3983 ])
3984
3985 ;;
3986 ;; [vsriq_n_u, vsriq_n_s])
3987 ;;
3988 (define_insn "mve_vsriq_n_<supf><mode>"
3989   [
3990    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3991         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3992                        (match_operand:MVE_2 2 "s_register_operand" "w")
3993                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3994          VSRIQ_N))
3995   ]
3996   "TARGET_HAVE_MVE"
3997   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3998   [(set_attr "type" "mve_move")
3999 ])
4000
4001 ;;
4002 ;; [vqdmlsdhxq_s])
4003 ;;
4004 (define_insn "mve_vqdmlsdhxq_s<mode>"
4005   [
4006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4008                        (match_operand:MVE_2 2 "s_register_operand" "w")
4009                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4010          VQDMLSDHXQ_S))
4011   ]
4012   "TARGET_HAVE_MVE"
4013   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4014   [(set_attr "type" "mve_move")
4015 ])
4016
4017 ;;
4018 ;; [vqdmlsdhq_s])
4019 ;;
4020 (define_insn "mve_vqdmlsdhq_s<mode>"
4021   [
4022    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4023         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4024                        (match_operand:MVE_2 2 "s_register_operand" "w")
4025                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4026          VQDMLSDHQ_S))
4027   ]
4028   "TARGET_HAVE_MVE"
4029   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4030   [(set_attr "type" "mve_move")
4031 ])
4032
4033 ;;
4034 ;; [vqdmladhxq_s])
4035 ;;
4036 (define_insn "mve_vqdmladhxq_s<mode>"
4037   [
4038    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4039         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4040                        (match_operand:MVE_2 2 "s_register_operand" "w")
4041                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4042          VQDMLADHXQ_S))
4043   ]
4044   "TARGET_HAVE_MVE"
4045   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4046   [(set_attr "type" "mve_move")
4047 ])
4048
4049 ;;
4050 ;; [vqdmladhq_s])
4051 ;;
4052 (define_insn "mve_vqdmladhq_s<mode>"
4053   [
4054    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4055         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4056                        (match_operand:MVE_2 2 "s_register_operand" "w")
4057                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4058          VQDMLADHQ_S))
4059   ]
4060   "TARGET_HAVE_MVE"
4061   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4062   [(set_attr "type" "mve_move")
4063 ])
4064
4065 ;;
4066 ;; [vmlsdavaxq_s])
4067 ;;
4068 (define_insn "mve_vmlsdavaxq_s<mode>"
4069   [
4070    (set (match_operand:SI 0 "s_register_operand" "=Te")
4071         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4072                     (match_operand:MVE_2 2 "s_register_operand" "w")
4073                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4074          VMLSDAVAXQ_S))
4075   ]
4076   "TARGET_HAVE_MVE"
4077   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4078   [(set_attr "type" "mve_move")
4079 ])
4080
4081 ;;
4082 ;; [vmlsdavaq_s])
4083 ;;
4084 (define_insn "mve_vmlsdavaq_s<mode>"
4085   [
4086    (set (match_operand:SI 0 "s_register_operand" "=Te")
4087         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4088                     (match_operand:MVE_2 2 "s_register_operand" "w")
4089                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4090          VMLSDAVAQ_S))
4091   ]
4092   "TARGET_HAVE_MVE"
4093   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4094   [(set_attr "type" "mve_move")
4095 ])
4096
4097 ;;
4098 ;; [vmladavaxq_s])
4099 ;;
4100 (define_insn "mve_vmladavaxq_s<mode>"
4101   [
4102    (set (match_operand:SI 0 "s_register_operand" "=Te")
4103         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4104                     (match_operand:MVE_2 2 "s_register_operand" "w")
4105                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4106          VMLADAVAXQ_S))
4107   ]
4108   "TARGET_HAVE_MVE"
4109   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4110   [(set_attr "type" "mve_move")
4111 ])
4112 ;;
4113 ;; [vabsq_m_f])
4114 ;;
4115 (define_insn "mve_vabsq_m_f<mode>"
4116   [
4117    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4118         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4119                        (match_operand:MVE_0 2 "s_register_operand" "w")
4120                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4121          VABSQ_M_F))
4122   ]
4123   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4124   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4125   [(set_attr "type" "mve_move")
4126    (set_attr "length""8")])
4127
4128 ;;
4129 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4130 ;;
4131 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4132   [
4133    (set (match_operand:DI 0 "s_register_operand" "=r")
4134         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4135                        (match_operand:V4SI 2 "s_register_operand" "w")
4136                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4137          VADDLVAQ_P))
4138   ]
4139   "TARGET_HAVE_MVE"
4140   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4141   [(set_attr "type" "mve_move")
4142    (set_attr "length""8")])
4143 ;;
4144 ;; [vcmlaq_f])
4145 ;;
4146 (define_insn "mve_vcmlaq_f<mode>"
4147   [
4148    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4149         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4150                        (match_operand:MVE_0 2 "s_register_operand" "w")
4151                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4152          VCMLAQ_F))
4153   ]
4154   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4155   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4156   [(set_attr "type" "mve_move")
4157 ])
4158
4159 ;;
4160 ;; [vcmlaq_rot180_f])
4161 ;;
4162 (define_insn "mve_vcmlaq_rot180_f<mode>"
4163   [
4164    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4165         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4166                        (match_operand:MVE_0 2 "s_register_operand" "w")
4167                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4168          VCMLAQ_ROT180_F))
4169   ]
4170   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4171   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4172   [(set_attr "type" "mve_move")
4173 ])
4174
4175 ;;
4176 ;; [vcmlaq_rot270_f])
4177 ;;
4178 (define_insn "mve_vcmlaq_rot270_f<mode>"
4179   [
4180    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4181         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4182                        (match_operand:MVE_0 2 "s_register_operand" "w")
4183                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4184          VCMLAQ_ROT270_F))
4185   ]
4186   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4187   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4188   [(set_attr "type" "mve_move")
4189 ])
4190
4191 ;;
4192 ;; [vcmlaq_rot90_f])
4193 ;;
4194 (define_insn "mve_vcmlaq_rot90_f<mode>"
4195   [
4196    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4197         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4198                        (match_operand:MVE_0 2 "s_register_operand" "w")
4199                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4200          VCMLAQ_ROT90_F))
4201   ]
4202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4203   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4204   [(set_attr "type" "mve_move")
4205 ])
4206
4207 ;;
4208 ;; [vcmpeqq_m_n_f])
4209 ;;
4210 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4211   [
4212    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4213         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4214                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4215                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4216          VCMPEQQ_M_N_F))
4217   ]
4218   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4219   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4220   [(set_attr "type" "mve_move")
4221    (set_attr "length""8")])
4222
4223 ;;
4224 ;; [vcmpgeq_m_f])
4225 ;;
4226 (define_insn "mve_vcmpgeq_m_f<mode>"
4227   [
4228    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4229         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4230                        (match_operand:MVE_0 2 "s_register_operand" "w")
4231                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4232          VCMPGEQ_M_F))
4233   ]
4234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4235   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4236   [(set_attr "type" "mve_move")
4237    (set_attr "length""8")])
4238
4239 ;;
4240 ;; [vcmpgeq_m_n_f])
4241 ;;
4242 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4243   [
4244    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4245         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4246                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4247                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4248          VCMPGEQ_M_N_F))
4249   ]
4250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4251   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4252   [(set_attr "type" "mve_move")
4253    (set_attr "length""8")])
4254
4255 ;;
4256 ;; [vcmpgtq_m_f])
4257 ;;
4258 (define_insn "mve_vcmpgtq_m_f<mode>"
4259   [
4260    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4261         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4262                        (match_operand:MVE_0 2 "s_register_operand" "w")
4263                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4264          VCMPGTQ_M_F))
4265   ]
4266   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4267   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4268   [(set_attr "type" "mve_move")
4269    (set_attr "length""8")])
4270
4271 ;;
4272 ;; [vcmpgtq_m_n_f])
4273 ;;
4274 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4275   [
4276    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4277         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4278                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4279                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4280          VCMPGTQ_M_N_F))
4281   ]
4282   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4283   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4284   [(set_attr "type" "mve_move")
4285    (set_attr "length""8")])
4286
4287 ;;
4288 ;; [vcmpleq_m_f])
4289 ;;
4290 (define_insn "mve_vcmpleq_m_f<mode>"
4291   [
4292    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4293         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4294                        (match_operand:MVE_0 2 "s_register_operand" "w")
4295                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4296          VCMPLEQ_M_F))
4297   ]
4298   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4299   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4300   [(set_attr "type" "mve_move")
4301    (set_attr "length""8")])
4302
4303 ;;
4304 ;; [vcmpleq_m_n_f])
4305 ;;
4306 (define_insn "mve_vcmpleq_m_n_f<mode>"
4307   [
4308    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4309         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4310                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4311                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4312          VCMPLEQ_M_N_F))
4313   ]
4314   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4315   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4316   [(set_attr "type" "mve_move")
4317    (set_attr "length""8")])
4318
4319 ;;
4320 ;; [vcmpltq_m_f])
4321 ;;
4322 (define_insn "mve_vcmpltq_m_f<mode>"
4323   [
4324    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4325         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4326                        (match_operand:MVE_0 2 "s_register_operand" "w")
4327                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4328          VCMPLTQ_M_F))
4329   ]
4330   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4331   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4332   [(set_attr "type" "mve_move")
4333    (set_attr "length""8")])
4334
4335 ;;
4336 ;; [vcmpltq_m_n_f])
4337 ;;
4338 (define_insn "mve_vcmpltq_m_n_f<mode>"
4339   [
4340    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4341         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4342                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4343                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4344          VCMPLTQ_M_N_F))
4345   ]
4346   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4347   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4348   [(set_attr "type" "mve_move")
4349    (set_attr "length""8")])
4350
4351 ;;
4352 ;; [vcmpneq_m_f])
4353 ;;
4354 (define_insn "mve_vcmpneq_m_f<mode>"
4355   [
4356    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4357         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4358                        (match_operand:MVE_0 2 "s_register_operand" "w")
4359                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4360          VCMPNEQ_M_F))
4361   ]
4362   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4363   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4364   [(set_attr "type" "mve_move")
4365    (set_attr "length""8")])
4366
4367 ;;
4368 ;; [vcmpneq_m_n_f])
4369 ;;
4370 (define_insn "mve_vcmpneq_m_n_f<mode>"
4371   [
4372    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4373         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4374                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4375                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4376          VCMPNEQ_M_N_F))
4377   ]
4378   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4379   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4380   [(set_attr "type" "mve_move")
4381    (set_attr "length""8")])
4382
4383 ;;
4384 ;; [vcvtbq_m_f16_f32])
4385 ;;
4386 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4387   [
4388    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4389         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4390                        (match_operand:V4SF 2 "s_register_operand" "w")
4391                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4392          VCVTBQ_M_F16_F32))
4393   ]
4394   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4395   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4396   [(set_attr "type" "mve_move")
4397    (set_attr "length""8")])
4398
4399 ;;
4400 ;; [vcvtbq_m_f32_f16])
4401 ;;
4402 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4403   [
4404    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4405         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4406                        (match_operand:V8HF 2 "s_register_operand" "w")
4407                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4408          VCVTBQ_M_F32_F16))
4409   ]
4410   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4411   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4412   [(set_attr "type" "mve_move")
4413    (set_attr "length""8")])
4414
4415 ;;
4416 ;; [vcvttq_m_f16_f32])
4417 ;;
4418 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4419   [
4420    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4421         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4422                        (match_operand:V4SF 2 "s_register_operand" "w")
4423                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4424          VCVTTQ_M_F16_F32))
4425   ]
4426   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4427   "vpst\;vcvttt.f16.f32 %q0, %q2"
4428   [(set_attr "type" "mve_move")
4429    (set_attr "length""8")])
4430
4431 ;;
4432 ;; [vcvttq_m_f32_f16])
4433 ;;
4434 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4435   [
4436    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4437         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4438                        (match_operand:V8HF 2 "s_register_operand" "w")
4439                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4440          VCVTTQ_M_F32_F16))
4441   ]
4442   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4443   "vpst\;vcvttt.f32.f16 %q0, %q2"
4444   [(set_attr "type" "mve_move")
4445    (set_attr "length""8")])
4446
4447 ;;
4448 ;; [vdupq_m_n_f])
4449 ;;
4450 (define_insn "mve_vdupq_m_n_f<mode>"
4451   [
4452    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4453         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4454                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4455                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4456          VDUPQ_M_N_F))
4457   ]
4458   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4459   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4460   [(set_attr "type" "mve_move")
4461    (set_attr "length""8")])
4462
4463 ;;
4464 ;; [vfmaq_f])
4465 ;;
4466 (define_insn "mve_vfmaq_f<mode>"
4467   [
4468    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4469         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4470                        (match_operand:MVE_0 2 "s_register_operand" "w")
4471                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4472          VFMAQ_F))
4473   ]
4474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4475   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4476   [(set_attr "type" "mve_move")
4477 ])
4478
4479 ;;
4480 ;; [vfmaq_n_f])
4481 ;;
4482 (define_insn "mve_vfmaq_n_f<mode>"
4483   [
4484    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4485         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4486                        (match_operand:MVE_0 2 "s_register_operand" "w")
4487                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4488          VFMAQ_N_F))
4489   ]
4490   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4491   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4492   [(set_attr "type" "mve_move")
4493 ])
4494
4495 ;;
4496 ;; [vfmasq_n_f])
4497 ;;
4498 (define_insn "mve_vfmasq_n_f<mode>"
4499   [
4500    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4501         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4502                        (match_operand:MVE_0 2 "s_register_operand" "w")
4503                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4504          VFMASQ_N_F))
4505   ]
4506   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4507   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4508   [(set_attr "type" "mve_move")
4509 ])
4510 ;;
4511 ;; [vfmsq_f])
4512 ;;
4513 (define_insn "mve_vfmsq_f<mode>"
4514   [
4515    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4516         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4517                        (match_operand:MVE_0 2 "s_register_operand" "w")
4518                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4519          VFMSQ_F))
4520   ]
4521   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4522   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4523   [(set_attr "type" "mve_move")
4524 ])
4525
4526 ;;
4527 ;; [vmaxnmaq_m_f])
4528 ;;
4529 (define_insn "mve_vmaxnmaq_m_f<mode>"
4530   [
4531    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4532         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4533                        (match_operand:MVE_0 2 "s_register_operand" "w")
4534                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4535          VMAXNMAQ_M_F))
4536   ]
4537   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4538   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4539   [(set_attr "type" "mve_move")
4540    (set_attr "length""8")])
4541 ;;
4542 ;; [vmaxnmavq_p_f])
4543 ;;
4544 (define_insn "mve_vmaxnmavq_p_f<mode>"
4545   [
4546    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4547         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4548                        (match_operand:MVE_0 2 "s_register_operand" "w")
4549                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4550          VMAXNMAVQ_P_F))
4551   ]
4552   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4553   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4554   [(set_attr "type" "mve_move")
4555    (set_attr "length""8")])
4556
4557 ;;
4558 ;; [vmaxnmvq_p_f])
4559 ;;
4560 (define_insn "mve_vmaxnmvq_p_f<mode>"
4561   [
4562    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4563         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4564                        (match_operand:MVE_0 2 "s_register_operand" "w")
4565                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4566          VMAXNMVQ_P_F))
4567   ]
4568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4569   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4570   [(set_attr "type" "mve_move")
4571    (set_attr "length""8")])
4572 ;;
4573 ;; [vminnmaq_m_f])
4574 ;;
4575 (define_insn "mve_vminnmaq_m_f<mode>"
4576   [
4577    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4578         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4579                        (match_operand:MVE_0 2 "s_register_operand" "w")
4580                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4581          VMINNMAQ_M_F))
4582   ]
4583   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4584   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4585   [(set_attr "type" "mve_move")
4586    (set_attr "length""8")])
4587
4588 ;;
4589 ;; [vminnmavq_p_f])
4590 ;;
4591 (define_insn "mve_vminnmavq_p_f<mode>"
4592   [
4593    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4594         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4595                        (match_operand:MVE_0 2 "s_register_operand" "w")
4596                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4597          VMINNMAVQ_P_F))
4598   ]
4599   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4600   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4601   [(set_attr "type" "mve_move")
4602    (set_attr "length""8")])
4603 ;;
4604 ;; [vminnmvq_p_f])
4605 ;;
4606 (define_insn "mve_vminnmvq_p_f<mode>"
4607   [
4608    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4609         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4610                        (match_operand:MVE_0 2 "s_register_operand" "w")
4611                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4612          VMINNMVQ_P_F))
4613   ]
4614   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4615   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4616   [(set_attr "type" "mve_move")
4617    (set_attr "length""8")])
4618
4619 ;;
4620 ;; [vmlaldavaq_s, vmlaldavaq_u])
4621 ;;
4622 (define_insn "mve_vmlaldavaq_<supf><mode>"
4623   [
4624    (set (match_operand:DI 0 "s_register_operand" "=r")
4625         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4626                        (match_operand:MVE_5 2 "s_register_operand" "w")
4627                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4628          VMLALDAVAQ))
4629   ]
4630   "TARGET_HAVE_MVE"
4631   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4632   [(set_attr "type" "mve_move")
4633 ])
4634
4635 ;;
4636 ;; [vmlaldavaxq_s])
4637 ;;
4638 (define_insn "mve_vmlaldavaxq_s<mode>"
4639   [
4640    (set (match_operand:DI 0 "s_register_operand" "=r")
4641         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4642                        (match_operand:MVE_5 2 "s_register_operand" "w")
4643                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4644          VMLALDAVAXQ_S))
4645   ]
4646   "TARGET_HAVE_MVE"
4647   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4648   [(set_attr "type" "mve_move")
4649 ])
4650
4651 ;;
4652 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4653 ;;
4654 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4655   [
4656    (set (match_operand:DI 0 "s_register_operand" "=r")
4657         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4658                        (match_operand:MVE_5 2 "s_register_operand" "w")
4659                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4660          VMLALDAVQ_P))
4661   ]
4662   "TARGET_HAVE_MVE"
4663   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4664   [(set_attr "type" "mve_move")
4665    (set_attr "length""8")])
4666
4667 ;;
4668 ;; [vmlaldavxq_p_s])
4669 ;;
4670 (define_insn "mve_vmlaldavxq_p_s<mode>"
4671   [
4672    (set (match_operand:DI 0 "s_register_operand" "=r")
4673         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4674                        (match_operand:MVE_5 2 "s_register_operand" "w")
4675                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4676          VMLALDAVXQ_P_S))
4677   ]
4678   "TARGET_HAVE_MVE"
4679   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4680   [(set_attr "type" "mve_move")
4681    (set_attr "length""8")])
4682 ;;
4683 ;; [vmlsldavaq_s])
4684 ;;
4685 (define_insn "mve_vmlsldavaq_s<mode>"
4686   [
4687    (set (match_operand:DI 0 "s_register_operand" "=r")
4688         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4689                        (match_operand:MVE_5 2 "s_register_operand" "w")
4690                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4691          VMLSLDAVAQ_S))
4692   ]
4693   "TARGET_HAVE_MVE"
4694   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4695   [(set_attr "type" "mve_move")
4696 ])
4697
4698 ;;
4699 ;; [vmlsldavaxq_s])
4700 ;;
4701 (define_insn "mve_vmlsldavaxq_s<mode>"
4702   [
4703    (set (match_operand:DI 0 "s_register_operand" "=r")
4704         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4705                        (match_operand:MVE_5 2 "s_register_operand" "w")
4706                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4707          VMLSLDAVAXQ_S))
4708   ]
4709   "TARGET_HAVE_MVE"
4710   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4711   [(set_attr "type" "mve_move")
4712 ])
4713
4714 ;;
4715 ;; [vmlsldavq_p_s])
4716 ;;
4717 (define_insn "mve_vmlsldavq_p_s<mode>"
4718   [
4719    (set (match_operand:DI 0 "s_register_operand" "=r")
4720         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4721                        (match_operand:MVE_5 2 "s_register_operand" "w")
4722                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4723          VMLSLDAVQ_P_S))
4724   ]
4725   "TARGET_HAVE_MVE"
4726   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4727   [(set_attr "type" "mve_move")
4728    (set_attr "length""8")])
4729
4730 ;;
4731 ;; [vmlsldavxq_p_s])
4732 ;;
4733 (define_insn "mve_vmlsldavxq_p_s<mode>"
4734   [
4735    (set (match_operand:DI 0 "s_register_operand" "=r")
4736         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4737                        (match_operand:MVE_5 2 "s_register_operand" "w")
4738                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4739          VMLSLDAVXQ_P_S))
4740   ]
4741   "TARGET_HAVE_MVE"
4742   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4743   [(set_attr "type" "mve_move")
4744    (set_attr "length""8")])
4745 ;;
4746 ;; [vmovlbq_m_u, vmovlbq_m_s])
4747 ;;
4748 (define_insn "mve_vmovlbq_m_<supf><mode>"
4749   [
4750    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4751         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4752                        (match_operand:MVE_3 2 "s_register_operand" "w")
4753                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4754          VMOVLBQ_M))
4755   ]
4756   "TARGET_HAVE_MVE"
4757   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4758   [(set_attr "type" "mve_move")
4759    (set_attr "length""8")])
4760 ;;
4761 ;; [vmovltq_m_u, vmovltq_m_s])
4762 ;;
4763 (define_insn "mve_vmovltq_m_<supf><mode>"
4764   [
4765    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4766         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4767                        (match_operand:MVE_3 2 "s_register_operand" "w")
4768                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4769          VMOVLTQ_M))
4770   ]
4771   "TARGET_HAVE_MVE"
4772   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4773   [(set_attr "type" "mve_move")
4774    (set_attr "length""8")])
4775 ;;
4776 ;; [vmovnbq_m_u, vmovnbq_m_s])
4777 ;;
4778 (define_insn "mve_vmovnbq_m_<supf><mode>"
4779   [
4780    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4781         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4782                        (match_operand:MVE_5 2 "s_register_operand" "w")
4783                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4784          VMOVNBQ_M))
4785   ]
4786   "TARGET_HAVE_MVE"
4787   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4788   [(set_attr "type" "mve_move")
4789    (set_attr "length""8")])
4790
4791 ;;
4792 ;; [vmovntq_m_u, vmovntq_m_s])
4793 ;;
4794 (define_insn "mve_vmovntq_m_<supf><mode>"
4795   [
4796    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4797         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4798                        (match_operand:MVE_5 2 "s_register_operand" "w")
4799                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4800          VMOVNTQ_M))
4801   ]
4802   "TARGET_HAVE_MVE"
4803   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4804   [(set_attr "type" "mve_move")
4805    (set_attr "length""8")])
4806
4807 ;;
4808 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4809 ;;
4810 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4811   [
4812    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4813         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4814                        (match_operand:SI 2 "immediate_operand" "i")
4815                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4816          VMVNQ_M_N))
4817   ]
4818   "TARGET_HAVE_MVE"
4819   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4820   [(set_attr "type" "mve_move")
4821    (set_attr "length""8")])
4822 ;;
4823 ;; [vnegq_m_f])
4824 ;;
4825 (define_insn "mve_vnegq_m_f<mode>"
4826   [
4827    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4828         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4829                        (match_operand:MVE_0 2 "s_register_operand" "w")
4830                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4831          VNEGQ_M_F))
4832   ]
4833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4834   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4835   [(set_attr "type" "mve_move")
4836    (set_attr "length""8")])
4837
4838 ;;
4839 ;; [vorrq_m_n_s, vorrq_m_n_u])
4840 ;;
4841 (define_insn "mve_vorrq_m_n_<supf><mode>"
4842   [
4843    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4844         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4845                        (match_operand:SI 2 "immediate_operand" "i")
4846                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4847          VORRQ_M_N))
4848   ]
4849   "TARGET_HAVE_MVE"
4850   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4851   [(set_attr "type" "mve_move")
4852    (set_attr "length""8")])
4853 ;;
4854 ;; [vpselq_f])
4855 ;;
4856 (define_insn "mve_vpselq_f<mode>"
4857   [
4858    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4859         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4860                        (match_operand:MVE_0 2 "s_register_operand" "w")
4861                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4862          VPSELQ_F))
4863   ]
4864   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4865   "vpsel %q0, %q1, %q2"
4866   [(set_attr "type" "mve_move")
4867 ])
4868
4869 ;;
4870 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4871 ;;
4872 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4873   [
4874    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4875         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4876                        (match_operand:MVE_5 2 "s_register_operand" "w")
4877                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4878          VQMOVNBQ_M))
4879   ]
4880   "TARGET_HAVE_MVE"
4881   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4882   [(set_attr "type" "mve_move")
4883    (set_attr "length""8")])
4884
4885 ;;
4886 ;; [vqmovntq_m_u, vqmovntq_m_s])
4887 ;;
4888 (define_insn "mve_vqmovntq_m_<supf><mode>"
4889   [
4890    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4891         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4892                        (match_operand:MVE_5 2 "s_register_operand" "w")
4893                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4894          VQMOVNTQ_M))
4895   ]
4896   "TARGET_HAVE_MVE"
4897   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4898   [(set_attr "type" "mve_move")
4899    (set_attr "length""8")])
4900
4901 ;;
4902 ;; [vqmovunbq_m_s])
4903 ;;
4904 (define_insn "mve_vqmovunbq_m_s<mode>"
4905   [
4906    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4907         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4908                        (match_operand:MVE_5 2 "s_register_operand" "w")
4909                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4910          VQMOVUNBQ_M_S))
4911   ]
4912   "TARGET_HAVE_MVE"
4913   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4914   [(set_attr "type" "mve_move")
4915    (set_attr "length""8")])
4916
4917 ;;
4918 ;; [vqmovuntq_m_s])
4919 ;;
4920 (define_insn "mve_vqmovuntq_m_s<mode>"
4921   [
4922    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4923         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4924                        (match_operand:MVE_5 2 "s_register_operand" "w")
4925                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4926          VQMOVUNTQ_M_S))
4927   ]
4928   "TARGET_HAVE_MVE"
4929   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4930   [(set_attr "type" "mve_move")
4931    (set_attr "length""8")])
4932
4933 ;;
4934 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4935 ;;
4936 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4937   [
4938    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4939         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4940                        (match_operand:MVE_5 2 "s_register_operand" "w")
4941                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4942          VQRSHRNTQ_N))
4943   ]
4944   "TARGET_HAVE_MVE"
4945   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4946   [(set_attr "type" "mve_move")
4947 ])
4948
4949 ;;
4950 ;; [vqrshruntq_n_s])
4951 ;;
4952 (define_insn "mve_vqrshruntq_n_s<mode>"
4953   [
4954    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4955         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4956                        (match_operand:MVE_5 2 "s_register_operand" "w")
4957                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4958          VQRSHRUNTQ_N_S))
4959   ]
4960   "TARGET_HAVE_MVE"
4961   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4962   [(set_attr "type" "mve_move")
4963 ])
4964
4965 ;;
4966 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4967 ;;
4968 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4969   [
4970    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4971         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4972                        (match_operand:MVE_5 2 "s_register_operand" "w")
4973                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4974          VQSHRNBQ_N))
4975   ]
4976   "TARGET_HAVE_MVE"
4977   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4978   [(set_attr "type" "mve_move")
4979 ])
4980
4981 ;;
4982 ;; [vqshrntq_n_u, vqshrntq_n_s])
4983 ;;
4984 (define_insn "mve_vqshrntq_n_<supf><mode>"
4985   [
4986    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4987         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4988                        (match_operand:MVE_5 2 "s_register_operand" "w")
4989                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4990          VQSHRNTQ_N))
4991   ]
4992   "TARGET_HAVE_MVE"
4993   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4994   [(set_attr "type" "mve_move")
4995 ])
4996
4997 ;;
4998 ;; [vqshrunbq_n_s])
4999 ;;
5000 (define_insn "mve_vqshrunbq_n_s<mode>"
5001   [
5002    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5003         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5004                        (match_operand:MVE_5 2 "s_register_operand" "w")
5005                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5006          VQSHRUNBQ_N_S))
5007   ]
5008   "TARGET_HAVE_MVE"
5009   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5010   [(set_attr "type" "mve_move")
5011 ])
5012
5013 ;;
5014 ;; [vqshruntq_n_s])
5015 ;;
5016 (define_insn "mve_vqshruntq_n_s<mode>"
5017   [
5018    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5019         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5020                        (match_operand:MVE_5 2 "s_register_operand" "w")
5021                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5022          VQSHRUNTQ_N_S))
5023   ]
5024   "TARGET_HAVE_MVE"
5025   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5026   [(set_attr "type" "mve_move")
5027 ])
5028
5029 ;;
5030 ;; [vrev32q_m_f])
5031 ;;
5032 (define_insn "mve_vrev32q_m_fv8hf"
5033   [
5034    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5035         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5036                        (match_operand:V8HF 2 "s_register_operand" "w")
5037                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5038          VREV32Q_M_F))
5039   ]
5040   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5041   "vpst\;vrev32t.16 %q0, %q2"
5042   [(set_attr "type" "mve_move")
5043    (set_attr "length""8")])
5044
5045 ;;
5046 ;; [vrev32q_m_s, vrev32q_m_u])
5047 ;;
5048 (define_insn "mve_vrev32q_m_<supf><mode>"
5049   [
5050    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5051         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5052                        (match_operand:MVE_3 2 "s_register_operand" "w")
5053                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5054          VREV32Q_M))
5055   ]
5056   "TARGET_HAVE_MVE"
5057   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5058   [(set_attr "type" "mve_move")
5059    (set_attr "length""8")])
5060
5061 ;;
5062 ;; [vrev64q_m_f])
5063 ;;
5064 (define_insn "mve_vrev64q_m_f<mode>"
5065   [
5066    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5067         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5068                        (match_operand:MVE_0 2 "s_register_operand" "w")
5069                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5070          VREV64Q_M_F))
5071   ]
5072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5073   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5074   [(set_attr "type" "mve_move")
5075    (set_attr "length""8")])
5076
5077 ;;
5078 ;; [vrmlaldavhaxq_s])
5079 ;;
5080 (define_insn "mve_vrmlaldavhaxq_sv4si"
5081   [
5082    (set (match_operand:DI 0 "s_register_operand" "=r")
5083         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5084                        (match_operand:V4SI 2 "s_register_operand" "w")
5085                        (match_operand:V4SI 3 "s_register_operand" "w")]
5086          VRMLALDAVHAXQ_S))
5087   ]
5088   "TARGET_HAVE_MVE"
5089   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5090   [(set_attr "type" "mve_move")
5091 ])
5092
5093 ;;
5094 ;; [vrmlaldavhxq_p_s])
5095 ;;
5096 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5097   [
5098    (set (match_operand:DI 0 "s_register_operand" "=r")
5099         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5100                        (match_operand:V4SI 2 "s_register_operand" "w")
5101                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5102          VRMLALDAVHXQ_P_S))
5103   ]
5104   "TARGET_HAVE_MVE"
5105   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5106   [(set_attr "type" "mve_move")
5107    (set_attr "length""8")])
5108
5109 ;;
5110 ;; [vrmlsldavhaxq_s])
5111 ;;
5112 (define_insn "mve_vrmlsldavhaxq_sv4si"
5113   [
5114    (set (match_operand:DI 0 "s_register_operand" "=r")
5115         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5116                        (match_operand:V4SI 2 "s_register_operand" "w")
5117                        (match_operand:V4SI 3 "s_register_operand" "w")]
5118          VRMLSLDAVHAXQ_S))
5119   ]
5120   "TARGET_HAVE_MVE"
5121   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5122   [(set_attr "type" "mve_move")
5123 ])
5124
5125 ;;
5126 ;; [vrmlsldavhq_p_s])
5127 ;;
5128 (define_insn "mve_vrmlsldavhq_p_sv4si"
5129   [
5130    (set (match_operand:DI 0 "s_register_operand" "=r")
5131         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5132                        (match_operand:V4SI 2 "s_register_operand" "w")
5133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5134          VRMLSLDAVHQ_P_S))
5135   ]
5136   "TARGET_HAVE_MVE"
5137   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5138   [(set_attr "type" "mve_move")
5139    (set_attr "length""8")])
5140
5141 ;;
5142 ;; [vrmlsldavhxq_p_s])
5143 ;;
5144 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5145   [
5146    (set (match_operand:DI 0 "s_register_operand" "=r")
5147         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5148                        (match_operand:V4SI 2 "s_register_operand" "w")
5149                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5150          VRMLSLDAVHXQ_P_S))
5151   ]
5152   "TARGET_HAVE_MVE"
5153   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5154   [(set_attr "type" "mve_move")
5155    (set_attr "length""8")])
5156
5157 ;;
5158 ;; [vrndaq_m_f])
5159 ;;
5160 (define_insn "mve_vrndaq_m_f<mode>"
5161   [
5162    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5163         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5164                        (match_operand:MVE_0 2 "s_register_operand" "w")
5165                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5166          VRNDAQ_M_F))
5167   ]
5168   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5169   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5170   [(set_attr "type" "mve_move")
5171    (set_attr "length""8")])
5172
5173 ;;
5174 ;; [vrndmq_m_f])
5175 ;;
5176 (define_insn "mve_vrndmq_m_f<mode>"
5177   [
5178    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5179         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5180                        (match_operand:MVE_0 2 "s_register_operand" "w")
5181                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5182          VRNDMQ_M_F))
5183   ]
5184   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5185   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5186   [(set_attr "type" "mve_move")
5187    (set_attr "length""8")])
5188
5189 ;;
5190 ;; [vrndnq_m_f])
5191 ;;
5192 (define_insn "mve_vrndnq_m_f<mode>"
5193   [
5194    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5195         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5196                        (match_operand:MVE_0 2 "s_register_operand" "w")
5197                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5198          VRNDNQ_M_F))
5199   ]
5200   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5201   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5202   [(set_attr "type" "mve_move")
5203    (set_attr "length""8")])
5204
5205 ;;
5206 ;; [vrndpq_m_f])
5207 ;;
5208 (define_insn "mve_vrndpq_m_f<mode>"
5209   [
5210    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5211         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5212                        (match_operand:MVE_0 2 "s_register_operand" "w")
5213                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5214          VRNDPQ_M_F))
5215   ]
5216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5217   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5218   [(set_attr "type" "mve_move")
5219    (set_attr "length""8")])
5220
5221 ;;
5222 ;; [vrndxq_m_f])
5223 ;;
5224 (define_insn "mve_vrndxq_m_f<mode>"
5225   [
5226    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5227         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5228                        (match_operand:MVE_0 2 "s_register_operand" "w")
5229                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5230          VRNDXQ_M_F))
5231   ]
5232   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5233   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5234   [(set_attr "type" "mve_move")
5235    (set_attr "length""8")])
5236
5237 ;;
5238 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5239 ;;
5240 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5241   [
5242    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5243         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5244                        (match_operand:MVE_5 2 "s_register_operand" "w")
5245                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5246          VRSHRNBQ_N))
5247   ]
5248   "TARGET_HAVE_MVE"
5249   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5250   [(set_attr "type" "mve_move")
5251 ])
5252
5253 ;;
5254 ;; [vrshrntq_n_u, vrshrntq_n_s])
5255 ;;
5256 (define_insn "mve_vrshrntq_n_<supf><mode>"
5257   [
5258    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5259         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5260                        (match_operand:MVE_5 2 "s_register_operand" "w")
5261                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5262          VRSHRNTQ_N))
5263   ]
5264   "TARGET_HAVE_MVE"
5265   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5266   [(set_attr "type" "mve_move")
5267 ])
5268
5269 ;;
5270 ;; [vshrnbq_n_u, vshrnbq_n_s])
5271 ;;
5272 (define_insn "mve_vshrnbq_n_<supf><mode>"
5273   [
5274    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5275         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5276                        (match_operand:MVE_5 2 "s_register_operand" "w")
5277                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5278          VSHRNBQ_N))
5279   ]
5280   "TARGET_HAVE_MVE"
5281   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5282   [(set_attr "type" "mve_move")
5283 ])
5284
5285 ;;
5286 ;; [vshrntq_n_s, vshrntq_n_u])
5287 ;;
5288 (define_insn "mve_vshrntq_n_<supf><mode>"
5289   [
5290    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5291         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5292                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5293                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5294          VSHRNTQ_N))
5295   ]
5296   "TARGET_HAVE_MVE"
5297   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5298   [(set_attr "type" "mve_move")
5299 ])
5300
5301 ;;
5302 ;; [vcvtmq_m_s, vcvtmq_m_u])
5303 ;;
5304 (define_insn "mve_vcvtmq_m_<supf><mode>"
5305   [
5306    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5307         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5308                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5309                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5310          VCVTMQ_M))
5311   ]
5312   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5313   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5314   [(set_attr "type" "mve_move")
5315    (set_attr "length""8")])
5316
5317 ;;
5318 ;; [vcvtpq_m_u, vcvtpq_m_s])
5319 ;;
5320 (define_insn "mve_vcvtpq_m_<supf><mode>"
5321   [
5322    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5323         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5324                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5325                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5326          VCVTPQ_M))
5327   ]
5328   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5329   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5330   [(set_attr "type" "mve_move")
5331    (set_attr "length""8")])
5332
5333 ;;
5334 ;; [vcvtnq_m_s, vcvtnq_m_u])
5335 ;;
5336 (define_insn "mve_vcvtnq_m_<supf><mode>"
5337   [
5338    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5339         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5340                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5341                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5342          VCVTNQ_M))
5343   ]
5344   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5345   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5346   [(set_attr "type" "mve_move")
5347    (set_attr "length""8")])
5348
5349 ;;
5350 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5351 ;;
5352 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5353   [
5354    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5355         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5356                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5357                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5358                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5359          VCVTQ_M_N_FROM_F))
5360   ]
5361   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5362   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5363   [(set_attr "type" "mve_move")
5364    (set_attr "length""8")])
5365
5366 ;;
5367 ;; [vrev16q_m_u, vrev16q_m_s])
5368 ;;
5369 (define_insn "mve_vrev16q_m_<supf>v16qi"
5370   [
5371    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5372         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5373                        (match_operand:V16QI 2 "s_register_operand" "w")
5374                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5375          VREV16Q_M))
5376   ]
5377   "TARGET_HAVE_MVE"
5378   "vpst\;vrev16t.8 %q0, %q2"
5379   [(set_attr "type" "mve_move")
5380    (set_attr "length""8")])
5381
5382 ;;
5383 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5384 ;;
5385 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5386   [
5387    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5388         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5389                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5390                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5391          VCVTQ_M_FROM_F))
5392   ]
5393   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5394   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5395   [(set_attr "type" "mve_move")
5396    (set_attr "length""8")])
5397
5398 ;;
5399 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5400 ;;
5401 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5402   [
5403    (set (match_operand:DI 0 "s_register_operand" "=r")
5404         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5405                     (match_operand:V4SI 2 "s_register_operand" "w")
5406                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5407          VRMLALDAVHQ_P))
5408   ]
5409   "TARGET_HAVE_MVE"
5410   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5411   [(set_attr "type" "mve_move")
5412    (set_attr "length""8")])
5413
5414 ;;
5415 ;; [vrmlsldavhaq_s])
5416 ;;
5417 (define_insn "mve_vrmlsldavhaq_sv4si"
5418   [
5419    (set (match_operand:DI 0 "s_register_operand" "=r")
5420         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5421                     (match_operand:V4SI 2 "s_register_operand" "w")
5422                     (match_operand:V4SI 3 "s_register_operand" "w")]
5423          VRMLSLDAVHAQ_S))
5424   ]
5425   "TARGET_HAVE_MVE"
5426   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5427   [(set_attr "type" "mve_move")
5428 ])
5429
5430 ;;
5431 ;; [vabavq_p_s, vabavq_p_u])
5432 ;;
5433 (define_insn "mve_vabavq_p_<supf><mode>"
5434   [
5435    (set (match_operand:SI 0 "s_register_operand" "=r")
5436         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5437                     (match_operand:MVE_2 2 "s_register_operand" "w")
5438                     (match_operand:MVE_2 3 "s_register_operand" "w")
5439                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5440          VABAVQ_P))
5441   ]
5442   "TARGET_HAVE_MVE"
5443   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5444   [(set_attr "type" "mve_move")
5445 ])
5446
5447 ;;
5448 ;; [vqshluq_m_n_s])
5449 ;;
5450 (define_insn "mve_vqshluq_m_n_s<mode>"
5451   [
5452    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5453         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5454                        (match_operand:MVE_2 2 "s_register_operand" "w")
5455                        (match_operand:SI 3 "mve_imm_7" "Ra")
5456                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5457          VQSHLUQ_M_N_S))
5458   ]
5459   "TARGET_HAVE_MVE"
5460   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5461   [(set_attr "type" "mve_move")])
5462
5463 ;;
5464 ;; [vshlq_m_s, vshlq_m_u])
5465 ;;
5466 (define_insn "mve_vshlq_m_<supf><mode>"
5467   [
5468    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5469         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5470                        (match_operand:MVE_2 2 "s_register_operand" "w")
5471                        (match_operand:MVE_2 3 "s_register_operand" "w")
5472                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5473          VSHLQ_M))
5474   ]
5475   "TARGET_HAVE_MVE"
5476   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5477   [(set_attr "type" "mve_move")])
5478
5479 ;;
5480 ;; [vsriq_m_n_s, vsriq_m_n_u])
5481 ;;
5482 (define_insn "mve_vsriq_m_n_<supf><mode>"
5483   [
5484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5486                        (match_operand:MVE_2 2 "s_register_operand" "w")
5487                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5489          VSRIQ_M_N))
5490   ]
5491   "TARGET_HAVE_MVE"
5492   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5493   [(set_attr "type" "mve_move")])
5494
5495 ;;
5496 ;; [vsubq_m_u, vsubq_m_s])
5497 ;;
5498 (define_insn "mve_vsubq_m_<supf><mode>"
5499   [
5500    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5501         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5502                        (match_operand:MVE_2 2 "s_register_operand" "w")
5503                        (match_operand:MVE_2 3 "s_register_operand" "w")
5504                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5505          VSUBQ_M))
5506   ]
5507   "TARGET_HAVE_MVE"
5508   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5509   [(set_attr "type" "mve_move")])
5510
5511 ;;
5512 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5513 ;;
5514 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5515   [
5516    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5517         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5518                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5519                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5520                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5521          VCVTQ_M_N_TO_F))
5522   ]
5523   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5524   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5525   [(set_attr "type" "mve_move")
5526    (set_attr "length""8")])
5527 ;;
5528 ;; [vabdq_m_s, vabdq_m_u])
5529 ;;
5530 (define_insn "mve_vabdq_m_<supf><mode>"
5531   [
5532    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5533         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5534                        (match_operand:MVE_2 2 "s_register_operand" "w")
5535                        (match_operand:MVE_2 3 "s_register_operand" "w")
5536                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5537          VABDQ_M))
5538   ]
5539   "TARGET_HAVE_MVE"
5540   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5541   [(set_attr "type" "mve_move")
5542    (set_attr "length""8")])
5543
5544 ;;
5545 ;; [vaddq_m_n_s, vaddq_m_n_u])
5546 ;;
5547 (define_insn "mve_vaddq_m_n_<supf><mode>"
5548   [
5549    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5550         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5551                        (match_operand:MVE_2 2 "s_register_operand" "w")
5552                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5553                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5554          VADDQ_M_N))
5555   ]
5556   "TARGET_HAVE_MVE"
5557   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5558   [(set_attr "type" "mve_move")
5559    (set_attr "length""8")])
5560
5561 ;;
5562 ;; [vaddq_m_u, vaddq_m_s])
5563 ;;
5564 (define_insn "mve_vaddq_m_<supf><mode>"
5565   [
5566    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5567         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5568                        (match_operand:MVE_2 2 "s_register_operand" "w")
5569                        (match_operand:MVE_2 3 "s_register_operand" "w")
5570                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5571          VADDQ_M))
5572   ]
5573   "TARGET_HAVE_MVE"
5574   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5575   [(set_attr "type" "mve_move")
5576    (set_attr "length""8")])
5577
5578 ;;
5579 ;; [vandq_m_u, vandq_m_s])
5580 ;;
5581 (define_insn "mve_vandq_m_<supf><mode>"
5582   [
5583    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5584         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5585                        (match_operand:MVE_2 2 "s_register_operand" "w")
5586                        (match_operand:MVE_2 3 "s_register_operand" "w")
5587                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5588          VANDQ_M))
5589   ]
5590   "TARGET_HAVE_MVE"
5591   "vpst\;vandt %q0, %q2, %q3"
5592   [(set_attr "type" "mve_move")
5593    (set_attr "length""8")])
5594
5595 ;;
5596 ;; [vbicq_m_u, vbicq_m_s])
5597 ;;
5598 (define_insn "mve_vbicq_m_<supf><mode>"
5599   [
5600    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5601         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5602                        (match_operand:MVE_2 2 "s_register_operand" "w")
5603                        (match_operand:MVE_2 3 "s_register_operand" "w")
5604                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5605          VBICQ_M))
5606   ]
5607   "TARGET_HAVE_MVE"
5608   "vpst\;vbict %q0, %q2, %q3"
5609   [(set_attr "type" "mve_move")
5610    (set_attr "length""8")])
5611
5612 ;;
5613 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5614 ;;
5615 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5616   [
5617    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5618         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5619                        (match_operand:MVE_2 2 "s_register_operand" "w")
5620                        (match_operand:SI 3 "s_register_operand" "r")
5621                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5622          VBRSRQ_M_N))
5623   ]
5624   "TARGET_HAVE_MVE"
5625   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5626   [(set_attr "type" "mve_move")
5627    (set_attr "length""8")])
5628
5629 ;;
5630 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5631 ;;
5632 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5633   [
5634    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5635         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5636                        (match_operand:MVE_2 2 "s_register_operand" "w")
5637                        (match_operand:MVE_2 3 "s_register_operand" "w")
5638                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5639          VCADDQ_ROT270_M))
5640   ]
5641   "TARGET_HAVE_MVE"
5642   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5643   [(set_attr "type" "mve_move")
5644    (set_attr "length""8")])
5645
5646 ;;
5647 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5648 ;;
5649 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5650   [
5651    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5652         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5653                        (match_operand:MVE_2 2 "s_register_operand" "w")
5654                        (match_operand:MVE_2 3 "s_register_operand" "w")
5655                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5656          VCADDQ_ROT90_M))
5657   ]
5658   "TARGET_HAVE_MVE"
5659   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5660   [(set_attr "type" "mve_move")
5661    (set_attr "length""8")])
5662
5663 ;;
5664 ;; [veorq_m_s, veorq_m_u])
5665 ;;
5666 (define_insn "mve_veorq_m_<supf><mode>"
5667   [
5668    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5669         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5670                        (match_operand:MVE_2 2 "s_register_operand" "w")
5671                        (match_operand:MVE_2 3 "s_register_operand" "w")
5672                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5673          VEORQ_M))
5674   ]
5675   "TARGET_HAVE_MVE"
5676   "vpst\;veort %q0, %q2, %q3"
5677   [(set_attr "type" "mve_move")
5678    (set_attr "length""8")])
5679
5680 ;;
5681 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5682 ;;
5683 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5684   [
5685    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5686         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5687                        (match_operand:MVE_2 2 "s_register_operand" "w")
5688                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5689                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5690          VHADDQ_M_N))
5691   ]
5692   "TARGET_HAVE_MVE"
5693   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5694   [(set_attr "type" "mve_move")
5695    (set_attr "length""8")])
5696
5697 ;;
5698 ;; [vhaddq_m_s, vhaddq_m_u])
5699 ;;
5700 (define_insn "mve_vhaddq_m_<supf><mode>"
5701   [
5702    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5703         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5704                        (match_operand:MVE_2 2 "s_register_operand" "w")
5705                        (match_operand:MVE_2 3 "s_register_operand" "w")
5706                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5707          VHADDQ_M))
5708   ]
5709   "TARGET_HAVE_MVE"
5710   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5711   [(set_attr "type" "mve_move")
5712    (set_attr "length""8")])
5713
5714 ;;
5715 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5716 ;;
5717 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5718   [
5719    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5720         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5721                        (match_operand:MVE_2 2 "s_register_operand" "w")
5722                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5723                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5724          VHSUBQ_M_N))
5725   ]
5726   "TARGET_HAVE_MVE"
5727   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5728   [(set_attr "type" "mve_move")
5729    (set_attr "length""8")])
5730
5731 ;;
5732 ;; [vhsubq_m_s, vhsubq_m_u])
5733 ;;
5734 (define_insn "mve_vhsubq_m_<supf><mode>"
5735   [
5736    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5737         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5738                        (match_operand:MVE_2 2 "s_register_operand" "w")
5739                        (match_operand:MVE_2 3 "s_register_operand" "w")
5740                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5741          VHSUBQ_M))
5742   ]
5743   "TARGET_HAVE_MVE"
5744   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5745   [(set_attr "type" "mve_move")
5746    (set_attr "length""8")])
5747
5748 ;;
5749 ;; [vmaxq_m_s, vmaxq_m_u])
5750 ;;
5751 (define_insn "mve_vmaxq_m_<supf><mode>"
5752   [
5753    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5754         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5755                        (match_operand:MVE_2 2 "s_register_operand" "w")
5756                        (match_operand:MVE_2 3 "s_register_operand" "w")
5757                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5758          VMAXQ_M))
5759   ]
5760   "TARGET_HAVE_MVE"
5761   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5762   [(set_attr "type" "mve_move")
5763    (set_attr "length""8")])
5764
5765 ;;
5766 ;; [vminq_m_s, vminq_m_u])
5767 ;;
5768 (define_insn "mve_vminq_m_<supf><mode>"
5769   [
5770    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5771         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5772                        (match_operand:MVE_2 2 "s_register_operand" "w")
5773                        (match_operand:MVE_2 3 "s_register_operand" "w")
5774                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5775          VMINQ_M))
5776   ]
5777   "TARGET_HAVE_MVE"
5778   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5779   [(set_attr "type" "mve_move")
5780    (set_attr "length""8")])
5781
5782 ;;
5783 ;; [vmladavaq_p_u, vmladavaq_p_s])
5784 ;;
5785 (define_insn "mve_vmladavaq_p_<supf><mode>"
5786   [
5787    (set (match_operand:SI 0 "s_register_operand" "=Te")
5788         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5789                     (match_operand:MVE_2 2 "s_register_operand" "w")
5790                     (match_operand:MVE_2 3 "s_register_operand" "w")
5791                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5792          VMLADAVAQ_P))
5793   ]
5794   "TARGET_HAVE_MVE"
5795   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5796   [(set_attr "type" "mve_move")
5797    (set_attr "length""8")])
5798
5799 ;;
5800 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5801 ;;
5802 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5803   [
5804    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5805         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5806                        (match_operand:MVE_2 2 "s_register_operand" "w")
5807                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5808                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5809          VMLAQ_M_N))
5810   ]
5811   "TARGET_HAVE_MVE"
5812   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5813   [(set_attr "type" "mve_move")
5814    (set_attr "length""8")])
5815
5816 ;;
5817 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5818 ;;
5819 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5820   [
5821    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5822         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5823                        (match_operand:MVE_2 2 "s_register_operand" "w")
5824                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5825                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5826          VMLASQ_M_N))
5827   ]
5828   "TARGET_HAVE_MVE"
5829   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5830   [(set_attr "type" "mve_move")
5831    (set_attr "length""8")])
5832
5833 ;;
5834 ;; [vmulhq_m_s, vmulhq_m_u])
5835 ;;
5836 (define_insn "mve_vmulhq_m_<supf><mode>"
5837   [
5838    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5839         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5840                        (match_operand:MVE_2 2 "s_register_operand" "w")
5841                        (match_operand:MVE_2 3 "s_register_operand" "w")
5842                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5843          VMULHQ_M))
5844   ]
5845   "TARGET_HAVE_MVE"
5846   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5847   [(set_attr "type" "mve_move")
5848    (set_attr "length""8")])
5849
5850 ;;
5851 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5852 ;;
5853 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5854   [
5855    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5856         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5857                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5858                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5859                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5860          VMULLBQ_INT_M))
5861   ]
5862   "TARGET_HAVE_MVE"
5863   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5864   [(set_attr "type" "mve_move")
5865    (set_attr "length""8")])
5866
5867 ;;
5868 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5869 ;;
5870 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5871   [
5872    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5873         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5874                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5875                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5876                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5877          VMULLTQ_INT_M))
5878   ]
5879   "TARGET_HAVE_MVE"
5880   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5881   [(set_attr "type" "mve_move")
5882    (set_attr "length""8")])
5883
5884 ;;
5885 ;; [vmulq_m_n_u, vmulq_m_n_s])
5886 ;;
5887 (define_insn "mve_vmulq_m_n_<supf><mode>"
5888   [
5889    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5890         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5891                        (match_operand:MVE_2 2 "s_register_operand" "w")
5892                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5893                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5894          VMULQ_M_N))
5895   ]
5896   "TARGET_HAVE_MVE"
5897   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5898   [(set_attr "type" "mve_move")
5899    (set_attr "length""8")])
5900
5901 ;;
5902 ;; [vmulq_m_s, vmulq_m_u])
5903 ;;
5904 (define_insn "mve_vmulq_m_<supf><mode>"
5905   [
5906    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5907         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5908                        (match_operand:MVE_2 2 "s_register_operand" "w")
5909                        (match_operand:MVE_2 3 "s_register_operand" "w")
5910                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5911          VMULQ_M))
5912   ]
5913   "TARGET_HAVE_MVE"
5914   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5915   [(set_attr "type" "mve_move")
5916    (set_attr "length""8")])
5917
5918 ;;
5919 ;; [vornq_m_u, vornq_m_s])
5920 ;;
5921 (define_insn "mve_vornq_m_<supf><mode>"
5922   [
5923    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5924         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5925                        (match_operand:MVE_2 2 "s_register_operand" "w")
5926                        (match_operand:MVE_2 3 "s_register_operand" "w")
5927                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5928          VORNQ_M))
5929   ]
5930   "TARGET_HAVE_MVE"
5931   "vpst\;vornt %q0, %q2, %q3"
5932   [(set_attr "type" "mve_move")
5933    (set_attr "length""8")])
5934
5935 ;;
5936 ;; [vorrq_m_s, vorrq_m_u])
5937 ;;
5938 (define_insn "mve_vorrq_m_<supf><mode>"
5939   [
5940    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5941         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5942                        (match_operand:MVE_2 2 "s_register_operand" "w")
5943                        (match_operand:MVE_2 3 "s_register_operand" "w")
5944                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5945          VORRQ_M))
5946   ]
5947   "TARGET_HAVE_MVE"
5948   "vpst\;vorrt %q0, %q2, %q3"
5949   [(set_attr "type" "mve_move")
5950    (set_attr "length""8")])
5951
5952 ;;
5953 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5954 ;;
5955 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5956   [
5957    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5958         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5959                        (match_operand:MVE_2 2 "s_register_operand" "w")
5960                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5961                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5962          VQADDQ_M_N))
5963   ]
5964   "TARGET_HAVE_MVE"
5965   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5966   [(set_attr "type" "mve_move")
5967    (set_attr "length""8")])
5968
5969 ;;
5970 ;; [vqaddq_m_u, vqaddq_m_s])
5971 ;;
5972 (define_insn "mve_vqaddq_m_<supf><mode>"
5973   [
5974    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5975         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5976                        (match_operand:MVE_2 2 "s_register_operand" "w")
5977                        (match_operand:MVE_2 3 "s_register_operand" "w")
5978                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5979          VQADDQ_M))
5980   ]
5981   "TARGET_HAVE_MVE"
5982   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5983   [(set_attr "type" "mve_move")
5984    (set_attr "length""8")])
5985
5986 ;;
5987 ;; [vqdmlahq_m_n_s])
5988 ;;
5989 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5990   [
5991    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5992         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5993                        (match_operand:MVE_2 2 "s_register_operand" "w")
5994                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5995                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5996          VQDMLAHQ_M_N_S))
5997   ]
5998   "TARGET_HAVE_MVE"
5999   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6000   [(set_attr "type" "mve_move")
6001    (set_attr "length""8")])
6002
6003 ;;
6004 ;; [vqdmlashq_m_n_s])
6005 ;;
6006 (define_insn "mve_vqdmlashq_m_n_s<mode>"
6007   [
6008    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6009         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6010                        (match_operand:MVE_2 2 "s_register_operand" "w")
6011                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6012                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6013          VQDMLASHQ_M_N_S))
6014   ]
6015   "TARGET_HAVE_MVE"
6016   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6017   [(set_attr "type" "mve_move")
6018    (set_attr "length""8")])
6019
6020 ;;
6021 ;; [vqrdmlahq_m_n_s])
6022 ;;
6023 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6024   [
6025    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6026         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6027                        (match_operand:MVE_2 2 "s_register_operand" "w")
6028                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6029                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6030          VQRDMLAHQ_M_N_S))
6031   ]
6032   "TARGET_HAVE_MVE"
6033   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6034   [(set_attr "type" "mve_move")
6035    (set_attr "length""8")])
6036
6037 ;;
6038 ;; [vqrdmlashq_m_n_s])
6039 ;;
6040 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6041   [
6042    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6043         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6044                        (match_operand:MVE_2 2 "s_register_operand" "w")
6045                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6046                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6047          VQRDMLASHQ_M_N_S))
6048   ]
6049   "TARGET_HAVE_MVE"
6050   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6051   [(set_attr "type" "mve_move")
6052    (set_attr "length""8")])
6053
6054 ;;
6055 ;; [vqrshlq_m_u, vqrshlq_m_s])
6056 ;;
6057 (define_insn "mve_vqrshlq_m_<supf><mode>"
6058   [
6059    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6060         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6061                        (match_operand:MVE_2 2 "s_register_operand" "w")
6062                        (match_operand:MVE_2 3 "s_register_operand" "w")
6063                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6064          VQRSHLQ_M))
6065   ]
6066   "TARGET_HAVE_MVE"
6067   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6068   [(set_attr "type" "mve_move")
6069    (set_attr "length""8")])
6070
6071 ;;
6072 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6073 ;;
6074 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6075   [
6076    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6077         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6078                        (match_operand:MVE_2 2 "s_register_operand" "w")
6079                        (match_operand:SI 3 "immediate_operand" "i")
6080                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6081          VQSHLQ_M_N))
6082   ]
6083   "TARGET_HAVE_MVE"
6084   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6085   [(set_attr "type" "mve_move")
6086    (set_attr "length""8")])
6087
6088 ;;
6089 ;; [vqshlq_m_u, vqshlq_m_s])
6090 ;;
6091 (define_insn "mve_vqshlq_m_<supf><mode>"
6092   [
6093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6095                        (match_operand:MVE_2 2 "s_register_operand" "w")
6096                        (match_operand:MVE_2 3 "s_register_operand" "w")
6097                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6098          VQSHLQ_M))
6099   ]
6100   "TARGET_HAVE_MVE"
6101   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6102   [(set_attr "type" "mve_move")
6103    (set_attr "length""8")])
6104
6105 ;;
6106 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6107 ;;
6108 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6109   [
6110    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6111         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6112                        (match_operand:MVE_2 2 "s_register_operand" "w")
6113                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6114                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6115          VQSUBQ_M_N))
6116   ]
6117   "TARGET_HAVE_MVE"
6118   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6119   [(set_attr "type" "mve_move")
6120    (set_attr "length""8")])
6121
6122 ;;
6123 ;; [vqsubq_m_u, vqsubq_m_s])
6124 ;;
6125 (define_insn "mve_vqsubq_m_<supf><mode>"
6126   [
6127    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6128         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6129                        (match_operand:MVE_2 2 "s_register_operand" "w")
6130                        (match_operand:MVE_2 3 "s_register_operand" "w")
6131                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6132          VQSUBQ_M))
6133   ]
6134   "TARGET_HAVE_MVE"
6135   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6136   [(set_attr "type" "mve_move")
6137    (set_attr "length""8")])
6138
6139 ;;
6140 ;; [vrhaddq_m_u, vrhaddq_m_s])
6141 ;;
6142 (define_insn "mve_vrhaddq_m_<supf><mode>"
6143   [
6144    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6145         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6146                        (match_operand:MVE_2 2 "s_register_operand" "w")
6147                        (match_operand:MVE_2 3 "s_register_operand" "w")
6148                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6149          VRHADDQ_M))
6150   ]
6151   "TARGET_HAVE_MVE"
6152   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6153   [(set_attr "type" "mve_move")
6154    (set_attr "length""8")])
6155
6156 ;;
6157 ;; [vrmulhq_m_u, vrmulhq_m_s])
6158 ;;
6159 (define_insn "mve_vrmulhq_m_<supf><mode>"
6160   [
6161    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6162         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6163                        (match_operand:MVE_2 2 "s_register_operand" "w")
6164                        (match_operand:MVE_2 3 "s_register_operand" "w")
6165                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6166          VRMULHQ_M))
6167   ]
6168   "TARGET_HAVE_MVE"
6169   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6170   [(set_attr "type" "mve_move")
6171    (set_attr "length""8")])
6172
6173 ;;
6174 ;; [vrshlq_m_s, vrshlq_m_u])
6175 ;;
6176 (define_insn "mve_vrshlq_m_<supf><mode>"
6177   [
6178    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6179         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6180                        (match_operand:MVE_2 2 "s_register_operand" "w")
6181                        (match_operand:MVE_2 3 "s_register_operand" "w")
6182                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6183          VRSHLQ_M))
6184   ]
6185   "TARGET_HAVE_MVE"
6186   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6187   [(set_attr "type" "mve_move")
6188    (set_attr "length""8")])
6189
6190 ;;
6191 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6192 ;;
6193 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6194   [
6195    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6196         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6197                        (match_operand:MVE_2 2 "s_register_operand" "w")
6198                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6199                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6200          VRSHRQ_M_N))
6201   ]
6202   "TARGET_HAVE_MVE"
6203   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6204   [(set_attr "type" "mve_move")
6205    (set_attr "length""8")])
6206
6207 ;;
6208 ;; [vshlq_m_n_s, vshlq_m_n_u])
6209 ;;
6210 (define_insn "mve_vshlq_m_n_<supf><mode>"
6211   [
6212    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6213         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6214                        (match_operand:MVE_2 2 "s_register_operand" "w")
6215                        (match_operand:SI 3 "immediate_operand" "i")
6216                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6217          VSHLQ_M_N))
6218   ]
6219   "TARGET_HAVE_MVE"
6220   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6221   [(set_attr "type" "mve_move")
6222    (set_attr "length""8")])
6223
6224 ;;
6225 ;; [vshrq_m_n_s, vshrq_m_n_u])
6226 ;;
6227 (define_insn "mve_vshrq_m_n_<supf><mode>"
6228   [
6229    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6230         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6231                        (match_operand:MVE_2 2 "s_register_operand" "w")
6232                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6233                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6234          VSHRQ_M_N))
6235   ]
6236   "TARGET_HAVE_MVE"
6237   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6238   [(set_attr "type" "mve_move")
6239    (set_attr "length""8")])
6240
6241 ;;
6242 ;; [vsliq_m_n_u, vsliq_m_n_s])
6243 ;;
6244 (define_insn "mve_vsliq_m_n_<supf><mode>"
6245    [
6246    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6247        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6248                        (match_operand:MVE_2 2 "s_register_operand" "w")
6249                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6250                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6251          VSLIQ_M_N))
6252   ]
6253   "TARGET_HAVE_MVE"
6254   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6255   [(set_attr "type" "mve_move")
6256    (set_attr "length""8")])
6257
6258 ;;
6259 ;; [vsubq_m_n_s, vsubq_m_n_u])
6260 ;;
6261 (define_insn "mve_vsubq_m_n_<supf><mode>"
6262   [
6263    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6264         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6265                        (match_operand:MVE_2 2 "s_register_operand" "w")
6266                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6267                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6268          VSUBQ_M_N))
6269   ]
6270   "TARGET_HAVE_MVE"
6271   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6272   [(set_attr "type" "mve_move")
6273    (set_attr "length""8")])
6274
6275 ;;
6276 ;; [vhcaddq_rot270_m_s])
6277 ;;
6278 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6279   [
6280    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6281         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6282                        (match_operand:MVE_2 2 "s_register_operand" "w")
6283                        (match_operand:MVE_2 3 "s_register_operand" "w")
6284                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6285          VHCADDQ_ROT270_M_S))
6286   ]
6287   "TARGET_HAVE_MVE"
6288   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6289   [(set_attr "type" "mve_move")
6290    (set_attr "length""8")])
6291
6292 ;;
6293 ;; [vhcaddq_rot90_m_s])
6294 ;;
6295 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6296   [
6297    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6298         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6299                        (match_operand:MVE_2 2 "s_register_operand" "w")
6300                        (match_operand:MVE_2 3 "s_register_operand" "w")
6301                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6302          VHCADDQ_ROT90_M_S))
6303   ]
6304   "TARGET_HAVE_MVE"
6305   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6306   [(set_attr "type" "mve_move")
6307    (set_attr "length""8")])
6308
6309 ;;
6310 ;; [vmladavaxq_p_s])
6311 ;;
6312 (define_insn "mve_vmladavaxq_p_s<mode>"
6313   [
6314    (set (match_operand:SI 0 "s_register_operand" "=Te")
6315         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6316                        (match_operand:MVE_2 2 "s_register_operand" "w")
6317                        (match_operand:MVE_2 3 "s_register_operand" "w")
6318                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6319          VMLADAVAXQ_P_S))
6320   ]
6321   "TARGET_HAVE_MVE"
6322   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6323   [(set_attr "type" "mve_move")
6324    (set_attr "length""8")])
6325
6326 ;;
6327 ;; [vmlsdavaq_p_s])
6328 ;;
6329 (define_insn "mve_vmlsdavaq_p_s<mode>"
6330   [
6331    (set (match_operand:SI 0 "s_register_operand" "=Te")
6332         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6333                        (match_operand:MVE_2 2 "s_register_operand" "w")
6334                        (match_operand:MVE_2 3 "s_register_operand" "w")
6335                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6336          VMLSDAVAQ_P_S))
6337   ]
6338   "TARGET_HAVE_MVE"
6339   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6340   [(set_attr "type" "mve_move")
6341    (set_attr "length""8")])
6342
6343 ;;
6344 ;; [vmlsdavaxq_p_s])
6345 ;;
6346 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6347   [
6348    (set (match_operand:SI 0 "s_register_operand" "=Te")
6349         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6350                        (match_operand:MVE_2 2 "s_register_operand" "w")
6351                        (match_operand:MVE_2 3 "s_register_operand" "w")
6352                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6353          VMLSDAVAXQ_P_S))
6354   ]
6355   "TARGET_HAVE_MVE"
6356   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6357   [(set_attr "type" "mve_move")
6358    (set_attr "length""8")])
6359
6360 ;;
6361 ;; [vqdmladhq_m_s])
6362 ;;
6363 (define_insn "mve_vqdmladhq_m_s<mode>"
6364   [
6365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6367                        (match_operand:MVE_2 2 "s_register_operand" "w")
6368                        (match_operand:MVE_2 3 "s_register_operand" "w")
6369                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6370          VQDMLADHQ_M_S))
6371   ]
6372   "TARGET_HAVE_MVE"
6373   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6374   [(set_attr "type" "mve_move")
6375    (set_attr "length""8")])
6376
6377 ;;
6378 ;; [vqdmladhxq_m_s])
6379 ;;
6380 (define_insn "mve_vqdmladhxq_m_s<mode>"
6381   [
6382    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6383         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6384                        (match_operand:MVE_2 2 "s_register_operand" "w")
6385                        (match_operand:MVE_2 3 "s_register_operand" "w")
6386                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6387          VQDMLADHXQ_M_S))
6388   ]
6389   "TARGET_HAVE_MVE"
6390   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6391   [(set_attr "type" "mve_move")
6392    (set_attr "length""8")])
6393
6394 ;;
6395 ;; [vqdmlsdhq_m_s])
6396 ;;
6397 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6398   [
6399    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6400         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6401                        (match_operand:MVE_2 2 "s_register_operand" "w")
6402                        (match_operand:MVE_2 3 "s_register_operand" "w")
6403                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6404          VQDMLSDHQ_M_S))
6405   ]
6406   "TARGET_HAVE_MVE"
6407   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6408   [(set_attr "type" "mve_move")
6409    (set_attr "length""8")])
6410
6411 ;;
6412 ;; [vqdmlsdhxq_m_s])
6413 ;;
6414 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6415   [
6416    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6417         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6418                        (match_operand:MVE_2 2 "s_register_operand" "w")
6419                        (match_operand:MVE_2 3 "s_register_operand" "w")
6420                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6421          VQDMLSDHXQ_M_S))
6422   ]
6423   "TARGET_HAVE_MVE"
6424   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6425   [(set_attr "type" "mve_move")
6426    (set_attr "length""8")])
6427
6428 ;;
6429 ;; [vqdmulhq_m_n_s])
6430 ;;
6431 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6432   [
6433    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6434         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6435                        (match_operand:MVE_2 2 "s_register_operand" "w")
6436                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6437                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6438          VQDMULHQ_M_N_S))
6439   ]
6440   "TARGET_HAVE_MVE"
6441   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6442   [(set_attr "type" "mve_move")
6443    (set_attr "length""8")])
6444
6445 ;;
6446 ;; [vqdmulhq_m_s])
6447 ;;
6448 (define_insn "mve_vqdmulhq_m_s<mode>"
6449   [
6450    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6451         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6452                        (match_operand:MVE_2 2 "s_register_operand" "w")
6453                        (match_operand:MVE_2 3 "s_register_operand" "w")
6454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6455          VQDMULHQ_M_S))
6456   ]
6457   "TARGET_HAVE_MVE"
6458   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6459   [(set_attr "type" "mve_move")
6460    (set_attr "length""8")])
6461
6462 ;;
6463 ;; [vqrdmladhq_m_s])
6464 ;;
6465 (define_insn "mve_vqrdmladhq_m_s<mode>"
6466   [
6467    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6468         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6469                        (match_operand:MVE_2 2 "s_register_operand" "w")
6470                        (match_operand:MVE_2 3 "s_register_operand" "w")
6471                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6472          VQRDMLADHQ_M_S))
6473   ]
6474   "TARGET_HAVE_MVE"
6475   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6476   [(set_attr "type" "mve_move")
6477    (set_attr "length""8")])
6478
6479 ;;
6480 ;; [vqrdmladhxq_m_s])
6481 ;;
6482 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6483   [
6484    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6485         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6486                        (match_operand:MVE_2 2 "s_register_operand" "w")
6487                        (match_operand:MVE_2 3 "s_register_operand" "w")
6488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6489          VQRDMLADHXQ_M_S))
6490   ]
6491   "TARGET_HAVE_MVE"
6492   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6493   [(set_attr "type" "mve_move")
6494    (set_attr "length""8")])
6495
6496 ;;
6497 ;; [vqrdmlsdhq_m_s])
6498 ;;
6499 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6500   [
6501    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6502         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6503                        (match_operand:MVE_2 2 "s_register_operand" "w")
6504                        (match_operand:MVE_2 3 "s_register_operand" "w")
6505                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6506          VQRDMLSDHQ_M_S))
6507   ]
6508   "TARGET_HAVE_MVE"
6509   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6510   [(set_attr "type" "mve_move")
6511    (set_attr "length""8")])
6512
6513 ;;
6514 ;; [vqrdmlsdhxq_m_s])
6515 ;;
6516 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6517   [
6518    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6519         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6520                        (match_operand:MVE_2 2 "s_register_operand" "w")
6521                        (match_operand:MVE_2 3 "s_register_operand" "w")
6522                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6523          VQRDMLSDHXQ_M_S))
6524   ]
6525   "TARGET_HAVE_MVE"
6526   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6527   [(set_attr "type" "mve_move")
6528    (set_attr "length""8")])
6529
6530 ;;
6531 ;; [vqrdmulhq_m_n_s])
6532 ;;
6533 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6534   [
6535    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6536         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6537                        (match_operand:MVE_2 2 "s_register_operand" "w")
6538                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6539                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6540          VQRDMULHQ_M_N_S))
6541   ]
6542   "TARGET_HAVE_MVE"
6543   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6544   [(set_attr "type" "mve_move")
6545    (set_attr "length""8")])
6546
6547 ;;
6548 ;; [vqrdmulhq_m_s])
6549 ;;
6550 (define_insn "mve_vqrdmulhq_m_s<mode>"
6551   [
6552    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6553         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6554                        (match_operand:MVE_2 2 "s_register_operand" "w")
6555                        (match_operand:MVE_2 3 "s_register_operand" "w")
6556                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6557          VQRDMULHQ_M_S))
6558   ]
6559   "TARGET_HAVE_MVE"
6560   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6561   [(set_attr "type" "mve_move")
6562    (set_attr "length""8")])
6563
6564 ;;
6565 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6566 ;;
6567 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6568   [
6569    (set (match_operand:DI 0 "s_register_operand" "=r")
6570         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6571                        (match_operand:MVE_5 2 "s_register_operand" "w")
6572                        (match_operand:MVE_5 3 "s_register_operand" "w")
6573                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6574          VMLALDAVAQ_P))
6575   ]
6576   "TARGET_HAVE_MVE"
6577   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6578   [(set_attr "type" "mve_move")
6579    (set_attr "length""8")])
6580
6581 ;;
6582 ;; [vmlaldavaxq_p_s])
6583 ;;
6584 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6585   [
6586    (set (match_operand:DI 0 "s_register_operand" "=r")
6587         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6588                        (match_operand:MVE_5 2 "s_register_operand" "w")
6589                        (match_operand:MVE_5 3 "s_register_operand" "w")
6590                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6591          VMLALDAVAXQ_P))
6592   ]
6593   "TARGET_HAVE_MVE"
6594   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6595   [(set_attr "type" "mve_move")
6596    (set_attr "length""8")])
6597
6598 ;;
6599 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6600 ;;
6601 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6602   [
6603    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6604         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6605                        (match_operand:MVE_5 2 "s_register_operand" "w")
6606                        (match_operand:SI 3 "mve_imm_8" "Rb")
6607                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6608          VQRSHRNBQ_M_N))
6609   ]
6610   "TARGET_HAVE_MVE"
6611   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6612   [(set_attr "type" "mve_move")
6613    (set_attr "length""8")])
6614
6615 ;;
6616 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6617 ;;
6618 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6619   [
6620    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6621         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6622                        (match_operand:MVE_5 2 "s_register_operand" "w")
6623                        (match_operand:SI 3 "mve_imm_8" "Rb")
6624                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6625          VQRSHRNTQ_M_N))
6626   ]
6627   "TARGET_HAVE_MVE"
6628   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6629   [(set_attr "type" "mve_move")
6630    (set_attr "length""8")])
6631
6632 ;;
6633 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6634 ;;
6635 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6636   [
6637    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6638         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6639                        (match_operand:MVE_5 2 "s_register_operand" "w")
6640                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6641                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6642          VQSHRNBQ_M_N))
6643   ]
6644   "TARGET_HAVE_MVE"
6645   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6646   [(set_attr "type" "mve_move")
6647    (set_attr "length""8")])
6648
6649 ;;
6650 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6651 ;;
6652 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6653   [
6654    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6655         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6656                        (match_operand:MVE_5 2 "s_register_operand" "w")
6657                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6658                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6659          VQSHRNTQ_M_N))
6660   ]
6661   "TARGET_HAVE_MVE"
6662   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6663   [(set_attr "type" "mve_move")
6664    (set_attr "length""8")])
6665
6666 ;;
6667 ;; [vrmlaldavhaq_p_s])
6668 ;;
6669 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6670   [
6671    (set (match_operand:DI 0 "s_register_operand" "=r")
6672         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6673                        (match_operand:V4SI 2 "s_register_operand" "w")
6674                        (match_operand:V4SI 3 "s_register_operand" "w")
6675                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6676          VRMLALDAVHAQ_P_S))
6677   ]
6678   "TARGET_HAVE_MVE"
6679   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6680   [(set_attr "type" "mve_move")
6681    (set_attr "length""8")])
6682
6683 ;;
6684 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6685 ;;
6686 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6687   [
6688    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6689         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6690                        (match_operand:MVE_5 2 "s_register_operand" "w")
6691                        (match_operand:SI 3 "mve_imm_8" "Rb")
6692                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6693          VRSHRNBQ_M_N))
6694   ]
6695   "TARGET_HAVE_MVE"
6696   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6697   [(set_attr "type" "mve_move")
6698    (set_attr "length""8")])
6699
6700 ;;
6701 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6702 ;;
6703 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6704   [
6705    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6706         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6707                        (match_operand:MVE_5 2 "s_register_operand" "w")
6708                        (match_operand:SI 3 "mve_imm_8" "Rb")
6709                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6710          VRSHRNTQ_M_N))
6711   ]
6712   "TARGET_HAVE_MVE"
6713   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6714   [(set_attr "type" "mve_move")
6715    (set_attr "length""8")])
6716
6717 ;;
6718 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6719 ;;
6720 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6721   [
6722    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6723         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6724                        (match_operand:MVE_3 2 "s_register_operand" "w")
6725                        (match_operand:SI 3 "immediate_operand" "i")
6726                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6727          VSHLLBQ_M_N))
6728   ]
6729   "TARGET_HAVE_MVE"
6730   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6731   [(set_attr "type" "mve_move")
6732    (set_attr "length""8")])
6733
6734 ;;
6735 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6736 ;;
6737 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6738   [
6739    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6740         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6741                        (match_operand:MVE_3 2 "s_register_operand" "w")
6742                        (match_operand:SI 3 "immediate_operand" "i")
6743                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6744          VSHLLTQ_M_N))
6745   ]
6746   "TARGET_HAVE_MVE"
6747   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6748   [(set_attr "type" "mve_move")
6749    (set_attr "length""8")])
6750
6751 ;;
6752 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6753 ;;
6754 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6755   [
6756    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6757         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6758                        (match_operand:MVE_5 2 "s_register_operand" "w")
6759                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6760                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6761          VSHRNBQ_M_N))
6762   ]
6763   "TARGET_HAVE_MVE"
6764   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6765   [(set_attr "type" "mve_move")
6766    (set_attr "length""8")])
6767
6768 ;;
6769 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6770 ;;
6771 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6772   [
6773    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6774         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6775                        (match_operand:MVE_5 2 "s_register_operand" "w")
6776                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6777                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6778          VSHRNTQ_M_N))
6779   ]
6780   "TARGET_HAVE_MVE"
6781   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6782   [(set_attr "type" "mve_move")
6783    (set_attr "length""8")])
6784
6785 ;;
6786 ;; [vmlsldavaq_p_s])
6787 ;;
6788 (define_insn "mve_vmlsldavaq_p_s<mode>"
6789   [
6790    (set (match_operand:DI 0 "s_register_operand" "=r")
6791         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6792                        (match_operand:MVE_5 2 "s_register_operand" "w")
6793                        (match_operand:MVE_5 3 "s_register_operand" "w")
6794                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6795          VMLSLDAVAQ_P_S))
6796   ]
6797   "TARGET_HAVE_MVE"
6798   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6799   [(set_attr "type" "mve_move")
6800    (set_attr "length""8")])
6801
6802 ;;
6803 ;; [vmlsldavaxq_p_s])
6804 ;;
6805 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6806   [
6807    (set (match_operand:DI 0 "s_register_operand" "=r")
6808         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6809                        (match_operand:MVE_5 2 "s_register_operand" "w")
6810                        (match_operand:MVE_5 3 "s_register_operand" "w")
6811                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6812          VMLSLDAVAXQ_P_S))
6813   ]
6814   "TARGET_HAVE_MVE"
6815   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6816   [(set_attr "type" "mve_move")
6817    (set_attr "length""8")])
6818
6819 ;;
6820 ;; [vmullbq_poly_m_p])
6821 ;;
6822 (define_insn "mve_vmullbq_poly_m_p<mode>"
6823   [
6824    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6825         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6826                        (match_operand:MVE_3 2 "s_register_operand" "w")
6827                        (match_operand:MVE_3 3 "s_register_operand" "w")
6828                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6829          VMULLBQ_POLY_M_P))
6830   ]
6831   "TARGET_HAVE_MVE"
6832   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6833   [(set_attr "type" "mve_move")
6834    (set_attr "length""8")])
6835
6836 ;;
6837 ;; [vmulltq_poly_m_p])
6838 ;;
6839 (define_insn "mve_vmulltq_poly_m_p<mode>"
6840   [
6841    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6842         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6843                        (match_operand:MVE_3 2 "s_register_operand" "w")
6844                        (match_operand:MVE_3 3 "s_register_operand" "w")
6845                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6846          VMULLTQ_POLY_M_P))
6847   ]
6848   "TARGET_HAVE_MVE"
6849   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6850   [(set_attr "type" "mve_move")
6851    (set_attr "length""8")])
6852
6853 ;;
6854 ;; [vqdmullbq_m_n_s])
6855 ;;
6856 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6857   [
6858    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6859         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6860                        (match_operand:MVE_5 2 "s_register_operand" "w")
6861                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6862                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6863          VQDMULLBQ_M_N_S))
6864   ]
6865   "TARGET_HAVE_MVE"
6866   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6867   [(set_attr "type" "mve_move")
6868    (set_attr "length""8")])
6869
6870 ;;
6871 ;; [vqdmullbq_m_s])
6872 ;;
6873 (define_insn "mve_vqdmullbq_m_s<mode>"
6874   [
6875    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6876         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6877                        (match_operand:MVE_5 2 "s_register_operand" "w")
6878                        (match_operand:MVE_5 3 "s_register_operand" "w")
6879                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6880          VQDMULLBQ_M_S))
6881   ]
6882   "TARGET_HAVE_MVE"
6883   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6884   [(set_attr "type" "mve_move")
6885    (set_attr "length""8")])
6886
6887 ;;
6888 ;; [vqdmulltq_m_n_s])
6889 ;;
6890 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6891   [
6892    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6893         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6894                        (match_operand:MVE_5 2 "s_register_operand" "w")
6895                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6896                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6897          VQDMULLTQ_M_N_S))
6898   ]
6899   "TARGET_HAVE_MVE"
6900   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6901   [(set_attr "type" "mve_move")
6902    (set_attr "length""8")])
6903
6904 ;;
6905 ;; [vqdmulltq_m_s])
6906 ;;
6907 (define_insn "mve_vqdmulltq_m_s<mode>"
6908   [
6909    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6910         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6911                        (match_operand:MVE_5 2 "s_register_operand" "w")
6912                        (match_operand:MVE_5 3 "s_register_operand" "w")
6913                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6914          VQDMULLTQ_M_S))
6915   ]
6916   "TARGET_HAVE_MVE"
6917   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6918   [(set_attr "type" "mve_move")
6919    (set_attr "length""8")])
6920
6921 ;;
6922 ;; [vqrshrunbq_m_n_s])
6923 ;;
6924 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6925   [
6926    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6927         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6928                        (match_operand:MVE_5 2 "s_register_operand" "w")
6929                        (match_operand:SI 3 "mve_imm_8" "Rb")
6930                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6931          VQRSHRUNBQ_M_N_S))
6932   ]
6933   "TARGET_HAVE_MVE"
6934   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6935   [(set_attr "type" "mve_move")
6936    (set_attr "length""8")])
6937
6938 ;;
6939 ;; [vqrshruntq_m_n_s])
6940 ;;
6941 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6942   [
6943    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6944         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6945                        (match_operand:MVE_5 2 "s_register_operand" "w")
6946                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6947                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6948          VQRSHRUNTQ_M_N_S))
6949   ]
6950   "TARGET_HAVE_MVE"
6951   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6952   [(set_attr "type" "mve_move")
6953    (set_attr "length""8")])
6954
6955 ;;
6956 ;; [vqshrunbq_m_n_s])
6957 ;;
6958 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6959   [
6960    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6961         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6962                        (match_operand:MVE_5 2 "s_register_operand" "w")
6963                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6964                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6965          VQSHRUNBQ_M_N_S))
6966   ]
6967   "TARGET_HAVE_MVE"
6968   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6969   [(set_attr "type" "mve_move")
6970    (set_attr "length""8")])
6971
6972 ;;
6973 ;; [vqshruntq_m_n_s])
6974 ;;
6975 (define_insn "mve_vqshruntq_m_n_s<mode>"
6976   [
6977    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6978         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6979                        (match_operand:MVE_5 2 "s_register_operand" "w")
6980                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6981                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6982          VQSHRUNTQ_M_N_S))
6983   ]
6984   "TARGET_HAVE_MVE"
6985   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6986   [(set_attr "type" "mve_move")
6987    (set_attr "length""8")])
6988
6989 ;;
6990 ;; [vrmlaldavhaq_p_u])
6991 ;;
6992 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6993   [
6994    (set (match_operand:DI 0 "s_register_operand" "=r")
6995         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6996                        (match_operand:V4SI 2 "s_register_operand" "w")
6997                        (match_operand:V4SI 3 "s_register_operand" "w")
6998                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6999          VRMLALDAVHAQ_P_U))
7000   ]
7001   "TARGET_HAVE_MVE"
7002   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7003   [(set_attr "type" "mve_move")
7004    (set_attr "length""8")])
7005
7006 ;;
7007 ;; [vrmlaldavhaxq_p_s])
7008 ;;
7009 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7010   [
7011    (set (match_operand:DI 0 "s_register_operand" "=r")
7012         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7013                        (match_operand:V4SI 2 "s_register_operand" "w")
7014                        (match_operand:V4SI 3 "s_register_operand" "w")
7015                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7016          VRMLALDAVHAXQ_P_S))
7017   ]
7018   "TARGET_HAVE_MVE"
7019   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7020   [(set_attr "type" "mve_move")
7021    (set_attr "length""8")])
7022
7023 ;;
7024 ;; [vrmlsldavhaq_p_s])
7025 ;;
7026 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7027   [
7028    (set (match_operand:DI 0 "s_register_operand" "=r")
7029         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7030                        (match_operand:V4SI 2 "s_register_operand" "w")
7031                        (match_operand:V4SI 3 "s_register_operand" "w")
7032                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7033          VRMLSLDAVHAQ_P_S))
7034   ]
7035   "TARGET_HAVE_MVE"
7036   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7037   [(set_attr "type" "mve_move")
7038    (set_attr "length""8")])
7039
7040 ;;
7041 ;; [vrmlsldavhaxq_p_s])
7042 ;;
7043 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7044   [
7045    (set (match_operand:DI 0 "s_register_operand" "=r")
7046         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7047                        (match_operand:V4SI 2 "s_register_operand" "w")
7048                        (match_operand:V4SI 3 "s_register_operand" "w")
7049                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7050          VRMLSLDAVHAXQ_P_S))
7051   ]
7052   "TARGET_HAVE_MVE"
7053   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7054   [(set_attr "type" "mve_move")
7055    (set_attr "length""8")])
7056 ;;
7057 ;; [vabdq_m_f])
7058 ;;
7059 (define_insn "mve_vabdq_m_f<mode>"
7060   [
7061    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7062         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7063                        (match_operand:MVE_0 2 "s_register_operand" "w")
7064                        (match_operand:MVE_0 3 "s_register_operand" "w")
7065                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7066          VABDQ_M_F))
7067   ]
7068   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7069   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7070   [(set_attr "type" "mve_move")
7071    (set_attr "length""8")])
7072
7073 ;;
7074 ;; [vaddq_m_f])
7075 ;;
7076 (define_insn "mve_vaddq_m_f<mode>"
7077   [
7078    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7079         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7080                        (match_operand:MVE_0 2 "s_register_operand" "w")
7081                        (match_operand:MVE_0 3 "s_register_operand" "w")
7082                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7083          VADDQ_M_F))
7084   ]
7085   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7086   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7087   [(set_attr "type" "mve_move")
7088    (set_attr "length""8")])
7089
7090 ;;
7091 ;; [vaddq_m_n_f])
7092 ;;
7093 (define_insn "mve_vaddq_m_n_f<mode>"
7094   [
7095    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7096         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7097                        (match_operand:MVE_0 2 "s_register_operand" "w")
7098                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7099                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7100          VADDQ_M_N_F))
7101   ]
7102   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7103   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7104   [(set_attr "type" "mve_move")
7105    (set_attr "length""8")])
7106
7107 ;;
7108 ;; [vandq_m_f])
7109 ;;
7110 (define_insn "mve_vandq_m_f<mode>"
7111   [
7112    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7113         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7114                        (match_operand:MVE_0 2 "s_register_operand" "w")
7115                        (match_operand:MVE_0 3 "s_register_operand" "w")
7116                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7117          VANDQ_M_F))
7118   ]
7119   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7120   "vpst\;vandt %q0, %q2, %q3"
7121   [(set_attr "type" "mve_move")
7122    (set_attr "length""8")])
7123
7124 ;;
7125 ;; [vbicq_m_f])
7126 ;;
7127 (define_insn "mve_vbicq_m_f<mode>"
7128   [
7129    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7130         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7131                        (match_operand:MVE_0 2 "s_register_operand" "w")
7132                        (match_operand:MVE_0 3 "s_register_operand" "w")
7133                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7134          VBICQ_M_F))
7135   ]
7136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7137   "vpst\;vbict %q0, %q2, %q3"
7138   [(set_attr "type" "mve_move")
7139    (set_attr "length""8")])
7140
7141 ;;
7142 ;; [vbrsrq_m_n_f])
7143 ;;
7144 (define_insn "mve_vbrsrq_m_n_f<mode>"
7145   [
7146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7148                        (match_operand:MVE_0 2 "s_register_operand" "w")
7149                        (match_operand:SI 3 "s_register_operand" "r")
7150                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7151          VBRSRQ_M_N_F))
7152   ]
7153   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7154   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7155   [(set_attr "type" "mve_move")
7156    (set_attr "length""8")])
7157
7158 ;;
7159 ;; [vcaddq_rot270_m_f])
7160 ;;
7161 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7162   [
7163    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7164         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7165                        (match_operand:MVE_0 2 "s_register_operand" "w")
7166                        (match_operand:MVE_0 3 "s_register_operand" "w")
7167                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7168          VCADDQ_ROT270_M_F))
7169   ]
7170   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7171   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7172   [(set_attr "type" "mve_move")
7173    (set_attr "length""8")])
7174
7175 ;;
7176 ;; [vcaddq_rot90_m_f])
7177 ;;
7178 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7179   [
7180    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7181         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7182                        (match_operand:MVE_0 2 "s_register_operand" "w")
7183                        (match_operand:MVE_0 3 "s_register_operand" "w")
7184                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7185          VCADDQ_ROT90_M_F))
7186   ]
7187   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7188   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7189   [(set_attr "type" "mve_move")
7190    (set_attr "length""8")])
7191
7192 ;;
7193 ;; [vcmlaq_m_f])
7194 ;;
7195 (define_insn "mve_vcmlaq_m_f<mode>"
7196   [
7197    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7198         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7199                        (match_operand:MVE_0 2 "s_register_operand" "w")
7200                        (match_operand:MVE_0 3 "s_register_operand" "w")
7201                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7202          VCMLAQ_M_F))
7203   ]
7204   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7205   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7206   [(set_attr "type" "mve_move")
7207    (set_attr "length""8")])
7208
7209 ;;
7210 ;; [vcmlaq_rot180_m_f])
7211 ;;
7212 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7213   [
7214    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7215         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7216                        (match_operand:MVE_0 2 "s_register_operand" "w")
7217                        (match_operand:MVE_0 3 "s_register_operand" "w")
7218                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7219          VCMLAQ_ROT180_M_F))
7220   ]
7221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7222   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7223   [(set_attr "type" "mve_move")
7224    (set_attr "length""8")])
7225
7226 ;;
7227 ;; [vcmlaq_rot270_m_f])
7228 ;;
7229 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7230   [
7231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7232         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7233                        (match_operand:MVE_0 2 "s_register_operand" "w")
7234                        (match_operand:MVE_0 3 "s_register_operand" "w")
7235                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7236          VCMLAQ_ROT270_M_F))
7237   ]
7238   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7239   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7240   [(set_attr "type" "mve_move")
7241    (set_attr "length""8")])
7242
7243 ;;
7244 ;; [vcmlaq_rot90_m_f])
7245 ;;
7246 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7247   [
7248    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7249         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7250                        (match_operand:MVE_0 2 "s_register_operand" "w")
7251                        (match_operand:MVE_0 3 "s_register_operand" "w")
7252                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7253          VCMLAQ_ROT90_M_F))
7254   ]
7255   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7256   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7257   [(set_attr "type" "mve_move")
7258    (set_attr "length""8")])
7259
7260 ;;
7261 ;; [vcmulq_m_f])
7262 ;;
7263 (define_insn "mve_vcmulq_m_f<mode>"
7264   [
7265    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7266         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7267                        (match_operand:MVE_0 2 "s_register_operand" "w")
7268                        (match_operand:MVE_0 3 "s_register_operand" "w")
7269                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7270          VCMULQ_M_F))
7271   ]
7272   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7273   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7274   [(set_attr "type" "mve_move")
7275    (set_attr "length""8")])
7276
7277 ;;
7278 ;; [vcmulq_rot180_m_f])
7279 ;;
7280 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7281   [
7282    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7283         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7284                        (match_operand:MVE_0 2 "s_register_operand" "w")
7285                        (match_operand:MVE_0 3 "s_register_operand" "w")
7286                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7287          VCMULQ_ROT180_M_F))
7288   ]
7289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7290   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7291   [(set_attr "type" "mve_move")
7292    (set_attr "length""8")])
7293
7294 ;;
7295 ;; [vcmulq_rot270_m_f])
7296 ;;
7297 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7298   [
7299    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7300         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7301                        (match_operand:MVE_0 2 "s_register_operand" "w")
7302                        (match_operand:MVE_0 3 "s_register_operand" "w")
7303                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7304          VCMULQ_ROT270_M_F))
7305   ]
7306   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7307   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7308   [(set_attr "type" "mve_move")
7309    (set_attr "length""8")])
7310
7311 ;;
7312 ;; [vcmulq_rot90_m_f])
7313 ;;
7314 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7315   [
7316    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7317         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7318                        (match_operand:MVE_0 2 "s_register_operand" "w")
7319                        (match_operand:MVE_0 3 "s_register_operand" "w")
7320                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7321          VCMULQ_ROT90_M_F))
7322   ]
7323   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7324   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7325   [(set_attr "type" "mve_move")
7326    (set_attr "length""8")])
7327
7328 ;;
7329 ;; [veorq_m_f])
7330 ;;
7331 (define_insn "mve_veorq_m_f<mode>"
7332   [
7333    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7334         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7335                        (match_operand:MVE_0 2 "s_register_operand" "w")
7336                        (match_operand:MVE_0 3 "s_register_operand" "w")
7337                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7338          VEORQ_M_F))
7339   ]
7340   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7341   "vpst\;veort %q0, %q2, %q3"
7342   [(set_attr "type" "mve_move")
7343    (set_attr "length""8")])
7344
7345 ;;
7346 ;; [vfmaq_m_f])
7347 ;;
7348 (define_insn "mve_vfmaq_m_f<mode>"
7349   [
7350    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7351         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7352                        (match_operand:MVE_0 2 "s_register_operand" "w")
7353                        (match_operand:MVE_0 3 "s_register_operand" "w")
7354                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7355          VFMAQ_M_F))
7356   ]
7357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7358   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7359   [(set_attr "type" "mve_move")
7360    (set_attr "length""8")])
7361
7362 ;;
7363 ;; [vfmaq_m_n_f])
7364 ;;
7365 (define_insn "mve_vfmaq_m_n_f<mode>"
7366   [
7367    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7368         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7369                        (match_operand:MVE_0 2 "s_register_operand" "w")
7370                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7371                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7372          VFMAQ_M_N_F))
7373   ]
7374   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7375   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7376   [(set_attr "type" "mve_move")
7377    (set_attr "length""8")])
7378
7379 ;;
7380 ;; [vfmasq_m_n_f])
7381 ;;
7382 (define_insn "mve_vfmasq_m_n_f<mode>"
7383   [
7384    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7385         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7386                        (match_operand:MVE_0 2 "s_register_operand" "w")
7387                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7388                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7389          VFMASQ_M_N_F))
7390   ]
7391   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7392   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7393   [(set_attr "type" "mve_move")
7394    (set_attr "length""8")])
7395
7396 ;;
7397 ;; [vfmsq_m_f])
7398 ;;
7399 (define_insn "mve_vfmsq_m_f<mode>"
7400   [
7401    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7402         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7403                        (match_operand:MVE_0 2 "s_register_operand" "w")
7404                        (match_operand:MVE_0 3 "s_register_operand" "w")
7405                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7406          VFMSQ_M_F))
7407   ]
7408   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7409   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7410   [(set_attr "type" "mve_move")
7411    (set_attr "length""8")])
7412
7413 ;;
7414 ;; [vmaxnmq_m_f])
7415 ;;
7416 (define_insn "mve_vmaxnmq_m_f<mode>"
7417   [
7418    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7419         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7420                        (match_operand:MVE_0 2 "s_register_operand" "w")
7421                        (match_operand:MVE_0 3 "s_register_operand" "w")
7422                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7423          VMAXNMQ_M_F))
7424   ]
7425   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7426   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7427   [(set_attr "type" "mve_move")
7428    (set_attr "length""8")])
7429
7430 ;;
7431 ;; [vminnmq_m_f])
7432 ;;
7433 (define_insn "mve_vminnmq_m_f<mode>"
7434   [
7435    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7436         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7437                        (match_operand:MVE_0 2 "s_register_operand" "w")
7438                        (match_operand:MVE_0 3 "s_register_operand" "w")
7439                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7440          VMINNMQ_M_F))
7441   ]
7442   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7443   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7444   [(set_attr "type" "mve_move")
7445    (set_attr "length""8")])
7446
7447 ;;
7448 ;; [vmulq_m_f])
7449 ;;
7450 (define_insn "mve_vmulq_m_f<mode>"
7451   [
7452    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7453         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7454                        (match_operand:MVE_0 2 "s_register_operand" "w")
7455                        (match_operand:MVE_0 3 "s_register_operand" "w")
7456                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7457          VMULQ_M_F))
7458   ]
7459   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7460   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7461   [(set_attr "type" "mve_move")
7462    (set_attr "length""8")])
7463
7464 ;;
7465 ;; [vmulq_m_n_f])
7466 ;;
7467 (define_insn "mve_vmulq_m_n_f<mode>"
7468   [
7469    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7470         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7471                        (match_operand:MVE_0 2 "s_register_operand" "w")
7472                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7473                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7474          VMULQ_M_N_F))
7475   ]
7476   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7477   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7478   [(set_attr "type" "mve_move")
7479    (set_attr "length""8")])
7480
7481 ;;
7482 ;; [vornq_m_f])
7483 ;;
7484 (define_insn "mve_vornq_m_f<mode>"
7485   [
7486    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7487         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7488                        (match_operand:MVE_0 2 "s_register_operand" "w")
7489                        (match_operand:MVE_0 3 "s_register_operand" "w")
7490                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7491          VORNQ_M_F))
7492   ]
7493   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7494   "vpst\;vornt %q0, %q2, %q3"
7495   [(set_attr "type" "mve_move")
7496    (set_attr "length""8")])
7497
7498 ;;
7499 ;; [vorrq_m_f])
7500 ;;
7501 (define_insn "mve_vorrq_m_f<mode>"
7502   [
7503    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7504         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7505                        (match_operand:MVE_0 2 "s_register_operand" "w")
7506                        (match_operand:MVE_0 3 "s_register_operand" "w")
7507                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7508          VORRQ_M_F))
7509   ]
7510   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7511   "vpst\;vorrt %q0, %q2, %q3"
7512   [(set_attr "type" "mve_move")
7513    (set_attr "length""8")])
7514
7515 ;;
7516 ;; [vsubq_m_f])
7517 ;;
7518 (define_insn "mve_vsubq_m_f<mode>"
7519   [
7520    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7521         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7522                        (match_operand:MVE_0 2 "s_register_operand" "w")
7523                        (match_operand:MVE_0 3 "s_register_operand" "w")
7524                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7525          VSUBQ_M_F))
7526   ]
7527   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7528   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7529   [(set_attr "type" "mve_move")
7530    (set_attr "length""8")])
7531
7532 ;;
7533 ;; [vsubq_m_n_f])
7534 ;;
7535 (define_insn "mve_vsubq_m_n_f<mode>"
7536   [
7537    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7538         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7539                        (match_operand:MVE_0 2 "s_register_operand" "w")
7540                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7541                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7542          VSUBQ_M_N_F))
7543   ]
7544   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7545   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7546   [(set_attr "type" "mve_move")
7547    (set_attr "length""8")])
7548
7549 ;;
7550 ;; [vstrbq_s vstrbq_u]
7551 ;;
7552 (define_insn "mve_vstrbq_<supf><mode>"
7553   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7554         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7555          VSTRBQ))
7556   ]
7557   "TARGET_HAVE_MVE"
7558 {
7559    rtx ops[2];
7560    int regno = REGNO (operands[1]);
7561    ops[1] = gen_rtx_REG (TImode, regno);
7562    ops[0]  = operands[0];
7563    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7564    return "";
7565 }
7566   [(set_attr "length" "4")])
7567
7568 ;;
7569 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7570 ;;
7571 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7572   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7573    (match_operand:MVE_2 1 "s_register_operand")
7574    (match_operand:MVE_2 2 "s_register_operand")
7575    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7576   "TARGET_HAVE_MVE"
7577 {
7578   rtx ind = XEXP (operands[0], 0);
7579   gcc_assert (REG_P (ind));
7580   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7581                                                               operands[2]));
7582   DONE;
7583 })
7584
7585 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7586   [(set (mem:BLK (scratch))
7587         (unspec:BLK
7588           [(match_operand:SI 0 "register_operand" "r")
7589            (match_operand:MVE_2 1 "s_register_operand" "w")
7590            (match_operand:MVE_2 2 "s_register_operand" "w")]
7591           VSTRBSOQ))]
7592   "TARGET_HAVE_MVE"
7593   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7594   [(set_attr "length" "4")])
7595
7596 ;;
7597 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7598 ;;
7599 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7600   [(set (mem:BLK (scratch))
7601         (unspec:BLK
7602                 [(match_operand:V4SI 0 "s_register_operand" "w")
7603                  (match_operand:SI 1 "immediate_operand" "i")
7604                  (match_operand:V4SI 2 "s_register_operand" "w")]
7605          VSTRWSBQ))
7606   ]
7607   "TARGET_HAVE_MVE"
7608 {
7609    rtx ops[3];
7610    ops[0] = operands[0];
7611    ops[1] = operands[1];
7612    ops[2] = operands[2];
7613    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7614    return "";
7615 }
7616   [(set_attr "length" "4")])
7617
7618 ;;
7619 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7620 ;;
7621 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7622   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7623         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7624                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7625          VLDRBGOQ))
7626   ]
7627   "TARGET_HAVE_MVE"
7628 {
7629    rtx ops[3];
7630    ops[0] = operands[0];
7631    ops[1] = operands[1];
7632    ops[2] = operands[2];
7633    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7634      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7635    else
7636      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7637    return "";
7638 }
7639   [(set_attr "length" "4")])
7640
7641 ;;
7642 ;; [vldrbq_s vldrbq_u]
7643 ;;
7644 (define_insn "mve_vldrbq_<supf><mode>"
7645   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7646         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7647          VLDRBQ))
7648   ]
7649   "TARGET_HAVE_MVE"
7650 {
7651    rtx ops[2];
7652    int regno = REGNO (operands[0]);
7653    ops[0] = gen_rtx_REG (TImode, regno);
7654    ops[1]  = operands[1];
7655    if (<V_sz_elem> == 8)
7656      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7657    else
7658      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7659    return "";
7660 }
7661   [(set_attr "length" "4")])
7662
7663 ;;
7664 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7665 ;;
7666 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7667   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7668         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7669                       (match_operand:SI 2 "immediate_operand" "i")]
7670          VLDRWGBQ))
7671   ]
7672   "TARGET_HAVE_MVE"
7673 {
7674    rtx ops[3];
7675    ops[0] = operands[0];
7676    ops[1] = operands[1];
7677    ops[2] = operands[2];
7678    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7679    return "";
7680 }
7681   [(set_attr "length" "4")])
7682
7683 ;;
7684 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7685 ;;
7686 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7687   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7688    (match_operand:MVE_2 1 "s_register_operand")
7689    (match_operand:MVE_2 2 "s_register_operand")
7690    (match_operand:HI 3 "vpr_register_operand" "Up")
7691    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7692   "TARGET_HAVE_MVE"
7693 {
7694   rtx ind = XEXP (operands[0], 0);
7695   gcc_assert (REG_P (ind));
7696   emit_insn (
7697     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7698                                                        operands[2],
7699                                                        operands[3]));
7700   DONE;
7701 })
7702
7703 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7704   [(set (mem:BLK (scratch))
7705         (unspec:BLK
7706           [(match_operand:SI 0 "register_operand" "r")
7707            (match_operand:MVE_2 1 "s_register_operand" "w")
7708            (match_operand:MVE_2 2 "s_register_operand" "w")
7709            (match_operand:HI 3 "vpr_register_operand" "Up")]
7710           VSTRBSOQ))]
7711   "TARGET_HAVE_MVE"
7712   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7713   [(set_attr "length" "8")])
7714
7715 ;;
7716 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7717 ;;
7718 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7719   [(set (mem:BLK (scratch))
7720         (unspec:BLK
7721                 [(match_operand:V4SI 0 "s_register_operand" "w")
7722                  (match_operand:SI 1 "immediate_operand" "i")
7723                  (match_operand:V4SI 2 "s_register_operand" "w")
7724                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7725          VSTRWSBQ))
7726   ]
7727   "TARGET_HAVE_MVE"
7728 {
7729    rtx ops[3];
7730    ops[0] = operands[0];
7731    ops[1] = operands[1];
7732    ops[2] = operands[2];
7733    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7734    return "";
7735 }
7736   [(set_attr "length" "8")])
7737
7738 ;;
7739 ;; [vstrbq_p_s vstrbq_p_u]
7740 ;;
7741 (define_insn "mve_vstrbq_p_<supf><mode>"
7742   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7743         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7744                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7745          VSTRBQ))
7746   ]
7747   "TARGET_HAVE_MVE"
7748 {
7749    rtx ops[2];
7750    int regno = REGNO (operands[1]);
7751    ops[1] = gen_rtx_REG (TImode, regno);
7752    ops[0]  = operands[0];
7753    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7754    return "";
7755 }
7756   [(set_attr "length" "8")])
7757
7758 ;;
7759 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7760 ;;
7761 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7762   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7763         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7764                        (match_operand:MVE_2 2 "s_register_operand" "w")
7765                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7766          VLDRBGOQ))
7767   ]
7768   "TARGET_HAVE_MVE"
7769 {
7770    rtx ops[4];
7771    ops[0] = operands[0];
7772    ops[1] = operands[1];
7773    ops[2] = operands[2];
7774    ops[3] = operands[3];
7775    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7776      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7777    else
7778      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7779    return "";
7780 }
7781   [(set_attr "length" "8")])
7782
7783 ;;
7784 ;; [vldrbq_z_s vldrbq_z_u]
7785 ;;
7786 (define_insn "mve_vldrbq_z_<supf><mode>"
7787   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7788         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7789                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7790          VLDRBQ))
7791   ]
7792   "TARGET_HAVE_MVE"
7793 {
7794    rtx ops[2];
7795    int regno = REGNO (operands[0]);
7796    ops[0] = gen_rtx_REG (TImode, regno);
7797    ops[1]  = operands[1];
7798    if (<V_sz_elem> == 8)
7799      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7800    else
7801      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7802    return "";
7803 }
7804   [(set_attr "length" "8")])
7805
7806 ;;
7807 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7808 ;;
7809 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7810   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7811         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7812                       (match_operand:SI 2 "immediate_operand" "i")
7813                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7814          VLDRWGBQ))
7815   ]
7816   "TARGET_HAVE_MVE"
7817 {
7818    rtx ops[3];
7819    ops[0] = operands[0];
7820    ops[1] = operands[1];
7821    ops[2] = operands[2];
7822    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7823    return "";
7824 }
7825   [(set_attr "length" "8")])
7826
7827 ;;
7828 ;; [vldrhq_f]
7829 ;;
7830 (define_insn "mve_vldrhq_fv8hf"
7831   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7832         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7833          VLDRHQ_F))
7834   ]
7835   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7836 {
7837    rtx ops[2];
7838    int regno = REGNO (operands[0]);
7839    ops[0] = gen_rtx_REG (TImode, regno);
7840    ops[1]  = operands[1];
7841    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7842    return "";
7843 }
7844   [(set_attr "length" "4")])
7845
7846 ;;
7847 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7848 ;;
7849 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7850   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7851         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7852                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7853         VLDRHGOQ))
7854   ]
7855   "TARGET_HAVE_MVE"
7856 {
7857    rtx ops[3];
7858    ops[0] = operands[0];
7859    ops[1] = operands[1];
7860    ops[2] = operands[2];
7861    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7862      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7863    else
7864      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7865    return "";
7866 }
7867   [(set_attr "length" "4")])
7868
7869 ;;
7870 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7871 ;;
7872 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7873   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7874         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7875                        (match_operand:MVE_6 2 "s_register_operand" "w")
7876                        (match_operand:HI 3 "vpr_register_operand" "Up")
7877         ]VLDRHGOQ))
7878   ]
7879   "TARGET_HAVE_MVE"
7880 {
7881    rtx ops[4];
7882    ops[0] = operands[0];
7883    ops[1] = operands[1];
7884    ops[2] = operands[2];
7885    ops[3] = operands[3];
7886    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7887      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7888    else
7889      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7890    return "";
7891 }
7892  [(set_attr "length" "8")])
7893
7894 ;;
7895 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7896 ;;
7897 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7898   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7899         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7900                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7901         VLDRHGSOQ))
7902   ]
7903   "TARGET_HAVE_MVE"
7904 {
7905    rtx ops[3];
7906    ops[0] = operands[0];
7907    ops[1] = operands[1];
7908    ops[2] = operands[2];
7909       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7910      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7911    else
7912      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7913    return "";
7914 }
7915   [(set_attr "length" "4")])
7916
7917 ;;
7918 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7919 ;;
7920 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7921   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7922         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7923                        (match_operand:MVE_6 2 "s_register_operand" "w")
7924                        (match_operand:HI 3 "vpr_register_operand" "Up")
7925         ]VLDRHGSOQ))
7926   ]
7927   "TARGET_HAVE_MVE"
7928 {
7929    rtx ops[4];
7930    ops[0] = operands[0];
7931    ops[1] = operands[1];
7932    ops[2] = operands[2];
7933    ops[3] = operands[3];
7934    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7935      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7936    else
7937      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7938    return "";
7939 }
7940   [(set_attr "length" "8")])
7941
7942 ;;
7943 ;; [vldrhq_s, vldrhq_u]
7944 ;;
7945 (define_insn "mve_vldrhq_<supf><mode>"
7946   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7947         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7948          VLDRHQ))
7949   ]
7950   "TARGET_HAVE_MVE"
7951 {
7952    rtx ops[2];
7953    int regno = REGNO (operands[0]);
7954    ops[0] = gen_rtx_REG (TImode, regno);
7955    ops[1]  = operands[1];
7956    if (<V_sz_elem> == 16)
7957      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7958    else
7959      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7960    return "";
7961 }
7962   [(set_attr "length" "4")])
7963
7964 ;;
7965 ;; [vldrhq_z_f]
7966 ;;
7967 (define_insn "mve_vldrhq_z_fv8hf"
7968   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7969         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7970         (match_operand:HI 2 "vpr_register_operand" "Up")]
7971          VLDRHQ_F))
7972   ]
7973   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7974 {
7975    rtx ops[2];
7976    int regno = REGNO (operands[0]);
7977    ops[0] = gen_rtx_REG (TImode, regno);
7978    ops[1]  = operands[1];
7979    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7980    return "";
7981 }
7982   [(set_attr "length" "8")])
7983
7984 ;;
7985 ;; [vldrhq_z_s vldrhq_z_u]
7986 ;;
7987 (define_insn "mve_vldrhq_z_<supf><mode>"
7988   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7989         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7990         (match_operand:HI 2 "vpr_register_operand" "Up")]
7991          VLDRHQ))
7992   ]
7993   "TARGET_HAVE_MVE"
7994 {
7995    rtx ops[2];
7996    int regno = REGNO (operands[0]);
7997    ops[0] = gen_rtx_REG (TImode, regno);
7998    ops[1]  = operands[1];
7999    if (<V_sz_elem> == 16)
8000      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
8001    else
8002      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8003    return "";
8004 }
8005   [(set_attr "length" "8")])
8006
8007 ;;
8008 ;; [vldrwq_f]
8009 ;;
8010 (define_insn "mve_vldrwq_fv4sf"
8011   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8012         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
8013          VLDRWQ_F))
8014   ]
8015   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8016 {
8017    rtx ops[2];
8018    int regno = REGNO (operands[0]);
8019    ops[0] = gen_rtx_REG (TImode, regno);
8020    ops[1]  = operands[1];
8021    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8022    return "";
8023 }
8024   [(set_attr "length" "4")])
8025
8026 ;;
8027 ;; [vldrwq_s vldrwq_u]
8028 ;;
8029 (define_insn "mve_vldrwq_<supf>v4si"
8030   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8031         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
8032          VLDRWQ))
8033   ]
8034   "TARGET_HAVE_MVE"
8035 {
8036    rtx ops[2];
8037    int regno = REGNO (operands[0]);
8038    ops[0] = gen_rtx_REG (TImode, regno);
8039    ops[1]  = operands[1];
8040    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8041    return "";
8042 }
8043   [(set_attr "length" "4")])
8044
8045 ;;
8046 ;; [vldrwq_z_f]
8047 ;;
8048 (define_insn "mve_vldrwq_z_fv4sf"
8049   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8050         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
8051         (match_operand:HI 2 "vpr_register_operand" "Up")]
8052          VLDRWQ_F))
8053   ]
8054   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8055 {
8056    rtx ops[2];
8057    int regno = REGNO (operands[0]);
8058    ops[0] = gen_rtx_REG (TImode, regno);
8059    ops[1]  = operands[1];
8060    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8061    return "";
8062 }
8063   [(set_attr "length" "8")])
8064
8065 ;;
8066 ;; [vldrwq_z_s vldrwq_z_u]
8067 ;;
8068 (define_insn "mve_vldrwq_z_<supf>v4si"
8069   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8070         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
8071         (match_operand:HI 2 "vpr_register_operand" "Up")]
8072          VLDRWQ))
8073   ]
8074   "TARGET_HAVE_MVE"
8075 {
8076    rtx ops[2];
8077    int regno = REGNO (operands[0]);
8078    ops[0] = gen_rtx_REG (TImode, regno);
8079    ops[1]  = operands[1];
8080    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8081    return "";
8082 }
8083   [(set_attr "length" "8")])
8084
8085 (define_expand "mve_vld1q_f<mode>"
8086   [(match_operand:MVE_0 0 "s_register_operand")
8087    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
8088   ]
8089   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8090 {
8091   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8092   DONE;
8093 })
8094
8095 (define_expand "mve_vld1q_<supf><mode>"
8096   [(match_operand:MVE_2 0 "s_register_operand")
8097    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
8098   ]
8099   "TARGET_HAVE_MVE"
8100 {
8101   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8102   DONE;
8103 })
8104
8105 ;;
8106 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8107 ;;
8108 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8109   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8110         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8111                       (match_operand:SI 2 "immediate_operand" "i")]
8112          VLDRDGBQ))
8113   ]
8114   "TARGET_HAVE_MVE"
8115 {
8116    rtx ops[3];
8117    ops[0] = operands[0];
8118    ops[1] = operands[1];
8119    ops[2] = operands[2];
8120    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8121    return "";
8122 }
8123   [(set_attr "length" "4")])
8124
8125 ;;
8126 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8127 ;;
8128 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8129   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8130         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8131                       (match_operand:SI 2 "immediate_operand" "i")
8132                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8133          VLDRDGBQ))
8134   ]
8135   "TARGET_HAVE_MVE"
8136 {
8137    rtx ops[3];
8138    ops[0] = operands[0];
8139    ops[1] = operands[1];
8140    ops[2] = operands[2];
8141    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8142    return "";
8143 }
8144   [(set_attr "length" "8")])
8145
8146 ;;
8147 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8148 ;;
8149 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8150  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8151        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8152                      (match_operand:V2DI 2 "s_register_operand" "w")]
8153         VLDRDGOQ))
8154  ]
8155  "TARGET_HAVE_MVE"
8156 {
8157   rtx ops[3];
8158   ops[0] = operands[0];
8159   ops[1] = operands[1];
8160   ops[2] = operands[2];
8161   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8162   return "";
8163 }
8164  [(set_attr "length" "4")])
8165
8166 ;;
8167 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8168 ;;
8169 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8170  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8171        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8172                      (match_operand:V2DI 2 "s_register_operand" "w")
8173                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8174         VLDRDGOQ))
8175  ]
8176  "TARGET_HAVE_MVE"
8177 {
8178   rtx ops[3];
8179   ops[0] = operands[0];
8180   ops[1] = operands[1];
8181   ops[2] = operands[2];
8182   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8183   return "";
8184 }
8185  [(set_attr "length" "8")])
8186
8187 ;;
8188 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8189 ;;
8190 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8191   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8192         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8193                       (match_operand:V2DI 2 "s_register_operand" "w")]
8194          VLDRDGSOQ))
8195   ]
8196   "TARGET_HAVE_MVE"
8197 {
8198    rtx ops[3];
8199    ops[0] = operands[0];
8200    ops[1] = operands[1];
8201    ops[2] = operands[2];
8202    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8203    return "";
8204 }
8205   [(set_attr "length" "4")])
8206
8207 ;;
8208 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8209 ;;
8210 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8211   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8212         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8213                       (match_operand:V2DI 2 "s_register_operand" "w")
8214                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8215          VLDRDGSOQ))
8216   ]
8217   "TARGET_HAVE_MVE"
8218 {
8219    rtx ops[3];
8220    ops[0] = operands[0];
8221    ops[1] = operands[1];
8222    ops[2] = operands[2];
8223    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8224    return "";
8225 }
8226   [(set_attr "length" "8")])
8227
8228 ;;
8229 ;; [vldrhq_gather_offset_f]
8230 ;;
8231 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8232   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8233         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8234                       (match_operand:V8HI 2 "s_register_operand" "w")]
8235          VLDRHQGO_F))
8236   ]
8237   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8238 {
8239    rtx ops[3];
8240    ops[0] = operands[0];
8241    ops[1] = operands[1];
8242    ops[2] = operands[2];
8243    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8244    return "";
8245 }
8246   [(set_attr "length" "4")])
8247
8248 ;;
8249 ;; [vldrhq_gather_offset_z_f]
8250 ;;
8251 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8252   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8253         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8254                       (match_operand:V8HI 2 "s_register_operand" "w")
8255                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8256          VLDRHQGO_F))
8257   ]
8258   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8259 {
8260    rtx ops[4];
8261    ops[0] = operands[0];
8262    ops[1] = operands[1];
8263    ops[2] = operands[2];
8264    ops[3] = operands[3];
8265    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8266    return "";
8267 }
8268   [(set_attr "length" "8")])
8269
8270 ;;
8271 ;; [vldrhq_gather_shifted_offset_f]
8272 ;;
8273 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8274   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8275         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8276                       (match_operand:V8HI 2 "s_register_operand" "w")]
8277          VLDRHQGSO_F))
8278   ]
8279   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8280 {
8281    rtx ops[3];
8282    ops[0] = operands[0];
8283    ops[1] = operands[1];
8284    ops[2] = operands[2];
8285    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8286    return "";
8287 }
8288   [(set_attr "length" "4")])
8289
8290 ;;
8291 ;; [vldrhq_gather_shifted_offset_z_f]
8292 ;;
8293 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8294   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8295         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8296                       (match_operand:V8HI 2 "s_register_operand" "w")
8297                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8298          VLDRHQGSO_F))
8299   ]
8300   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8301 {
8302    rtx ops[4];
8303    ops[0] = operands[0];
8304    ops[1] = operands[1];
8305    ops[2] = operands[2];
8306    ops[3] = operands[3];
8307    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8308    return "";
8309 }
8310   [(set_attr "length" "8")])
8311
8312 ;;
8313 ;; [vldrwq_gather_base_f]
8314 ;;
8315 (define_insn "mve_vldrwq_gather_base_fv4sf"
8316   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8317         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8318                       (match_operand:SI 2 "immediate_operand" "i")]
8319          VLDRWQGB_F))
8320   ]
8321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8322 {
8323    rtx ops[3];
8324    ops[0] = operands[0];
8325    ops[1] = operands[1];
8326    ops[2] = operands[2];
8327    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8328    return "";
8329 }
8330   [(set_attr "length" "4")])
8331
8332 ;;
8333 ;; [vldrwq_gather_base_z_f]
8334 ;;
8335 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8336   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8337         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8338                       (match_operand:SI 2 "immediate_operand" "i")
8339                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8340          VLDRWQGB_F))
8341   ]
8342   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8343 {
8344    rtx ops[3];
8345    ops[0] = operands[0];
8346    ops[1] = operands[1];
8347    ops[2] = operands[2];
8348    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8349    return "";
8350 }
8351   [(set_attr "length" "8")])
8352
8353 ;;
8354 ;; [vldrwq_gather_offset_f]
8355 ;;
8356 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8357   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8358         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8359                        (match_operand:V4SI 2 "s_register_operand" "w")]
8360          VLDRWQGO_F))
8361   ]
8362   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8363 {
8364    rtx ops[3];
8365    ops[0] = operands[0];
8366    ops[1] = operands[1];
8367    ops[2] = operands[2];
8368    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8369    return "";
8370 }
8371   [(set_attr "length" "4")])
8372
8373 ;;
8374 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8375 ;;
8376 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8377   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8378         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8379                        (match_operand:V4SI 2 "s_register_operand" "w")]
8380          VLDRWGOQ))
8381   ]
8382   "TARGET_HAVE_MVE"
8383 {
8384    rtx ops[3];
8385    ops[0] = operands[0];
8386    ops[1] = operands[1];
8387    ops[2] = operands[2];
8388    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8389    return "";
8390 }
8391   [(set_attr "length" "4")])
8392
8393 ;;
8394 ;; [vldrwq_gather_offset_z_f]
8395 ;;
8396 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8397   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8398         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8399                       (match_operand:V4SI 2 "s_register_operand" "w")
8400                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8401          VLDRWQGO_F))
8402   ]
8403   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8404 {
8405    rtx ops[4];
8406    ops[0] = operands[0];
8407    ops[1] = operands[1];
8408    ops[2] = operands[2];
8409    ops[3] = operands[3];
8410    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8411    return "";
8412 }
8413   [(set_attr "length" "8")])
8414
8415 ;;
8416 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8417 ;;
8418 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8419   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8420         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8421                       (match_operand:V4SI 2 "s_register_operand" "w")
8422                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8423          VLDRWGOQ))
8424   ]
8425   "TARGET_HAVE_MVE"
8426 {
8427    rtx ops[4];
8428    ops[0] = operands[0];
8429    ops[1] = operands[1];
8430    ops[2] = operands[2];
8431    ops[3] = operands[3];
8432    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8433    return "";
8434 }
8435   [(set_attr "length" "8")])
8436
8437 ;;
8438 ;; [vldrwq_gather_shifted_offset_f]
8439 ;;
8440 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8441   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8442         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8443                       (match_operand:V4SI 2 "s_register_operand" "w")]
8444          VLDRWQGSO_F))
8445   ]
8446   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8447 {
8448    rtx ops[3];
8449    ops[0] = operands[0];
8450    ops[1] = operands[1];
8451    ops[2] = operands[2];
8452    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8453    return "";
8454 }
8455   [(set_attr "length" "4")])
8456
8457 ;;
8458 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8459 ;;
8460 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8461   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8462         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8463                       (match_operand:V4SI 2 "s_register_operand" "w")]
8464          VLDRWGSOQ))
8465   ]
8466   "TARGET_HAVE_MVE"
8467 {
8468    rtx ops[3];
8469    ops[0] = operands[0];
8470    ops[1] = operands[1];
8471    ops[2] = operands[2];
8472    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8473    return "";
8474 }
8475   [(set_attr "length" "4")])
8476
8477 ;;
8478 ;; [vldrwq_gather_shifted_offset_z_f]
8479 ;;
8480 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8481   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8482         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8483                       (match_operand:V4SI 2 "s_register_operand" "w")
8484                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8485          VLDRWQGSO_F))
8486   ]
8487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8488 {
8489    rtx ops[4];
8490    ops[0] = operands[0];
8491    ops[1] = operands[1];
8492    ops[2] = operands[2];
8493    ops[3] = operands[3];
8494    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8495    return "";
8496 }
8497   [(set_attr "length" "8")])
8498
8499 ;;
8500 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8501 ;;
8502 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8503   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8504         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8505                       (match_operand:V4SI 2 "s_register_operand" "w")
8506                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8507          VLDRWGSOQ))
8508   ]
8509   "TARGET_HAVE_MVE"
8510 {
8511    rtx ops[4];
8512    ops[0] = operands[0];
8513    ops[1] = operands[1];
8514    ops[2] = operands[2];
8515    ops[3] = operands[3];
8516    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8517    return "";
8518 }
8519   [(set_attr "length" "8")])
8520
8521 ;;
8522 ;; [vstrhq_f]
8523 ;;
8524 (define_insn "mve_vstrhq_fv8hf"
8525   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8526         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8527          VSTRHQ_F))
8528   ]
8529   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8530 {
8531    rtx ops[2];
8532    int regno = REGNO (operands[1]);
8533    ops[1] = gen_rtx_REG (TImode, regno);
8534    ops[0]  = operands[0];
8535    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8536    return "";
8537 }
8538   [(set_attr "length" "4")])
8539
8540 ;;
8541 ;; [vstrhq_p_f]
8542 ;;
8543 (define_insn "mve_vstrhq_p_fv8hf"
8544   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8545         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8546                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8547          VSTRHQ_F))
8548   ]
8549   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8550 {
8551    rtx ops[2];
8552    int regno = REGNO (operands[1]);
8553    ops[1] = gen_rtx_REG (TImode, regno);
8554    ops[0]  = operands[0];
8555    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8556    return "";
8557 }
8558   [(set_attr "length" "8")])
8559
8560 ;;
8561 ;; [vstrhq_p_s vstrhq_p_u]
8562 ;;
8563 (define_insn "mve_vstrhq_p_<supf><mode>"
8564   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8565         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8566                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8567          VSTRHQ))
8568   ]
8569   "TARGET_HAVE_MVE"
8570 {
8571    rtx ops[2];
8572    int regno = REGNO (operands[1]);
8573    ops[1] = gen_rtx_REG (TImode, regno);
8574    ops[0]  = operands[0];
8575    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8576    return "";
8577 }
8578   [(set_attr "length" "8")])
8579
8580 ;;
8581 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8582 ;;
8583 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8584   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8585    (match_operand:MVE_6 1 "s_register_operand")
8586    (match_operand:MVE_6 2 "s_register_operand")
8587    (match_operand:HI 3 "vpr_register_operand")
8588    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8589   "TARGET_HAVE_MVE"
8590 {
8591   rtx ind = XEXP (operands[0], 0);
8592   gcc_assert (REG_P (ind));
8593   emit_insn (
8594     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8595                                                        operands[2],
8596                                                        operands[3]));
8597   DONE;
8598 })
8599
8600 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8601   [(set (mem:BLK (scratch))
8602         (unspec:BLK
8603           [(match_operand:SI 0 "register_operand" "r")
8604            (match_operand:MVE_6 1 "s_register_operand" "w")
8605            (match_operand:MVE_6 2 "s_register_operand" "w")
8606            (match_operand:HI 3 "vpr_register_operand" "Up")]
8607           VSTRHSOQ))]
8608   "TARGET_HAVE_MVE"
8609   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8610   [(set_attr "length" "8")])
8611
8612 ;;
8613 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8614 ;;
8615 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8616   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8617    (match_operand:MVE_6 1 "s_register_operand")
8618    (match_operand:MVE_6 2 "s_register_operand")
8619    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8620   "TARGET_HAVE_MVE"
8621 {
8622   rtx ind = XEXP (operands[0], 0);
8623   gcc_assert (REG_P (ind));
8624   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8625                                                               operands[2]));
8626   DONE;
8627 })
8628
8629 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8630   [(set (mem:BLK (scratch))
8631         (unspec:BLK
8632           [(match_operand:SI 0 "register_operand" "r")
8633            (match_operand:MVE_6 1 "s_register_operand" "w")
8634            (match_operand:MVE_6 2 "s_register_operand" "w")]
8635           VSTRHSOQ))]
8636   "TARGET_HAVE_MVE"
8637   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8638   [(set_attr "length" "4")])
8639
8640 ;;
8641 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8642 ;;
8643 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8644   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8645    (match_operand:MVE_6 1 "s_register_operand")
8646    (match_operand:MVE_6 2 "s_register_operand")
8647    (match_operand:HI 3 "vpr_register_operand")
8648    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8649   "TARGET_HAVE_MVE"
8650 {
8651   rtx ind = XEXP (operands[0], 0);
8652   gcc_assert (REG_P (ind));
8653   emit_insn (
8654     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8655                                                                operands[2],
8656                                                                operands[3]));
8657   DONE;
8658 })
8659
8660 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8661   [(set (mem:BLK (scratch))
8662         (unspec:BLK
8663           [(match_operand:SI 0 "register_operand" "r")
8664            (match_operand:MVE_6 1 "s_register_operand" "w")
8665            (match_operand:MVE_6 2 "s_register_operand" "w")
8666            (match_operand:HI 3 "vpr_register_operand" "Up")]
8667           VSTRHSSOQ))]
8668   "TARGET_HAVE_MVE"
8669   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8670   [(set_attr "length" "8")])
8671
8672 ;;
8673 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8674 ;;
8675 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8676   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8677    (match_operand:MVE_6 1 "s_register_operand")
8678    (match_operand:MVE_6 2 "s_register_operand")
8679    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8680   "TARGET_HAVE_MVE"
8681 {
8682   rtx ind = XEXP (operands[0], 0);
8683   gcc_assert (REG_P (ind));
8684   emit_insn (
8685     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8686                                                              operands[2]));
8687   DONE;
8688 })
8689
8690 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8691   [(set (mem:BLK (scratch))
8692         (unspec:BLK
8693           [(match_operand:SI 0 "register_operand" "r")
8694            (match_operand:MVE_6 1 "s_register_operand" "w")
8695            (match_operand:MVE_6 2 "s_register_operand" "w")]
8696           VSTRHSSOQ))]
8697   "TARGET_HAVE_MVE"
8698   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8699   [(set_attr "length" "4")])
8700
8701 ;;
8702 ;; [vstrhq_s, vstrhq_u]
8703 ;;
8704 (define_insn "mve_vstrhq_<supf><mode>"
8705   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8706         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8707          VSTRHQ))
8708   ]
8709   "TARGET_HAVE_MVE"
8710 {
8711    rtx ops[2];
8712    int regno = REGNO (operands[1]);
8713    ops[1] = gen_rtx_REG (TImode, regno);
8714    ops[0]  = operands[0];
8715    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8716    return "";
8717 }
8718   [(set_attr "length" "4")])
8719
8720 ;;
8721 ;; [vstrwq_f]
8722 ;;
8723 (define_insn "mve_vstrwq_fv4sf"
8724   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8725         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8726          VSTRWQ_F))
8727   ]
8728   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8729 {
8730    rtx ops[2];
8731    int regno = REGNO (operands[1]);
8732    ops[1] = gen_rtx_REG (TImode, regno);
8733    ops[0]  = operands[0];
8734    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8735    return "";
8736 }
8737   [(set_attr "length" "4")])
8738
8739 ;;
8740 ;; [vstrwq_p_f]
8741 ;;
8742 (define_insn "mve_vstrwq_p_fv4sf"
8743   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8744         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8745                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8746          VSTRWQ_F))
8747   ]
8748   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8749 {
8750    rtx ops[2];
8751    int regno = REGNO (operands[1]);
8752    ops[1] = gen_rtx_REG (TImode, regno);
8753    ops[0]  = operands[0];
8754    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8755    return "";
8756 }
8757   [(set_attr "length" "8")])
8758
8759 ;;
8760 ;; [vstrwq_p_s vstrwq_p_u]
8761 ;;
8762 (define_insn "mve_vstrwq_p_<supf>v4si"
8763   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8764         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8765                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8766          VSTRWQ))
8767   ]
8768   "TARGET_HAVE_MVE"
8769 {
8770    rtx ops[2];
8771    int regno = REGNO (operands[1]);
8772    ops[1] = gen_rtx_REG (TImode, regno);
8773    ops[0]  = operands[0];
8774    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8775    return "";
8776 }
8777   [(set_attr "length" "8")])
8778
8779 ;;
8780 ;; [vstrwq_s vstrwq_u]
8781 ;;
8782 (define_insn "mve_vstrwq_<supf>v4si"
8783   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8784         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8785          VSTRWQ))
8786   ]
8787   "TARGET_HAVE_MVE"
8788 {
8789    rtx ops[2];
8790    int regno = REGNO (operands[1]);
8791    ops[1] = gen_rtx_REG (TImode, regno);
8792    ops[0]  = operands[0];
8793    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8794    return "";
8795 }
8796   [(set_attr "length" "4")])
8797
8798 (define_expand "mve_vst1q_f<mode>"
8799   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8800    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8801   ]
8802   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8803 {
8804   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8805   DONE;
8806 })
8807
8808 (define_expand "mve_vst1q_<supf><mode>"
8809   [(match_operand:MVE_2 0 "mve_memory_operand")
8810    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8811   ]
8812   "TARGET_HAVE_MVE"
8813 {
8814   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8815   DONE;
8816 })
8817
8818 ;;
8819 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8820 ;;
8821 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8822   [(set (mem:BLK (scratch))
8823         (unspec:BLK
8824                 [(match_operand:V2DI 0 "s_register_operand" "w")
8825                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8826                  (match_operand:V2DI 2 "s_register_operand" "w")
8827                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8828          VSTRDSBQ))
8829   ]
8830   "TARGET_HAVE_MVE"
8831 {
8832    rtx ops[3];
8833    ops[0] = operands[0];
8834    ops[1] = operands[1];
8835    ops[2] = operands[2];
8836    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8837    return "";
8838 }
8839   [(set_attr "length" "8")])
8840
8841 ;;
8842 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8843 ;;
8844 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8845   [(set (mem:BLK (scratch))
8846         (unspec:BLK
8847                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8848                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8849                  (match_operand:V2DI 2 "s_register_operand" "w")]
8850          VSTRDSBQ))
8851   ]
8852   "TARGET_HAVE_MVE"
8853 {
8854    rtx ops[3];
8855    ops[0] = operands[0];
8856    ops[1] = operands[1];
8857    ops[2] = operands[2];
8858    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8859    return "";
8860 }
8861   [(set_attr "length" "4")])
8862
8863 ;;
8864 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8865 ;;
8866 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8867   [(match_operand:V2DI 0 "mve_scatter_memory")
8868    (match_operand:V2DI 1 "s_register_operand")
8869    (match_operand:V2DI 2 "s_register_operand")
8870    (match_operand:HI 3 "vpr_register_operand")
8871    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8872   "TARGET_HAVE_MVE"
8873 {
8874   rtx ind = XEXP (operands[0], 0);
8875   gcc_assert (REG_P (ind));
8876   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8877                                                               operands[2],
8878                                                               operands[3]));
8879   DONE;
8880 })
8881
8882 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8883   [(set (mem:BLK (scratch))
8884         (unspec:BLK
8885           [(match_operand:SI 0 "register_operand" "r")
8886            (match_operand:V2DI 1 "s_register_operand" "w")
8887            (match_operand:V2DI 2 "s_register_operand" "w")
8888            (match_operand:HI 3 "vpr_register_operand" "Up")]
8889           VSTRDSOQ))]
8890   "TARGET_HAVE_MVE"
8891   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8892   [(set_attr "length" "8")])
8893
8894 ;;
8895 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8896 ;;
8897 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8898   [(match_operand:V2DI 0 "mve_scatter_memory")
8899    (match_operand:V2DI 1 "s_register_operand")
8900    (match_operand:V2DI 2 "s_register_operand")
8901    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8902   "TARGET_HAVE_MVE"
8903 {
8904   rtx ind = XEXP (operands[0], 0);
8905   gcc_assert (REG_P (ind));
8906   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8907                                                             operands[2]));
8908   DONE;
8909 })
8910
8911 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8912   [(set (mem:BLK (scratch))
8913         (unspec:BLK
8914           [(match_operand:SI 0 "register_operand" "r")
8915            (match_operand:V2DI 1 "s_register_operand" "w")
8916            (match_operand:V2DI 2 "s_register_operand" "w")]
8917           VSTRDSOQ))]
8918   "TARGET_HAVE_MVE"
8919   "vstrd.64\t%q2, [%0, %q1]"
8920   [(set_attr "length" "4")])
8921
8922 ;;
8923 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8924 ;;
8925 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8926   [(match_operand:V2DI 0 "mve_scatter_memory")
8927    (match_operand:V2DI 1 "s_register_operand")
8928    (match_operand:V2DI 2 "s_register_operand")
8929    (match_operand:HI 3 "vpr_register_operand")
8930    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8931   "TARGET_HAVE_MVE"
8932 {
8933   rtx ind = XEXP (operands[0], 0);
8934   gcc_assert (REG_P (ind));
8935   emit_insn (
8936     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8937                                                              operands[2],
8938                                                              operands[3]));
8939   DONE;
8940 })
8941
8942 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8943   [(set (mem:BLK (scratch))
8944         (unspec:BLK
8945           [(match_operand:SI 0 "register_operand" "r")
8946            (match_operand:V2DI 1 "s_register_operand" "w")
8947            (match_operand:V2DI 2 "s_register_operand" "w")
8948            (match_operand:HI 3 "vpr_register_operand" "Up")]
8949           VSTRDSSOQ))]
8950   "TARGET_HAVE_MVE"
8951   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8952   [(set_attr "length" "8")])
8953
8954 ;;
8955 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8956 ;;
8957 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8958   [(match_operand:V2DI 0 "mve_scatter_memory")
8959    (match_operand:V2DI 1 "s_register_operand")
8960    (match_operand:V2DI 2 "s_register_operand")
8961    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8962   "TARGET_HAVE_MVE"
8963 {
8964   rtx ind = XEXP (operands[0], 0);
8965   gcc_assert (REG_P (ind));
8966   emit_insn (
8967     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8968                                                            operands[2]));
8969   DONE;
8970 })
8971
8972 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8973   [(set (mem:BLK (scratch))
8974         (unspec:BLK
8975           [(match_operand:SI 0 "register_operand" "r")
8976            (match_operand:V2DI 1 "s_register_operand" "w")
8977            (match_operand:V2DI 2 "s_register_operand" "w")]
8978           VSTRDSSOQ))]
8979   "TARGET_HAVE_MVE"
8980   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8981   [(set_attr "length" "4")])
8982
8983 ;;
8984 ;; [vstrhq_scatter_offset_f]
8985 ;;
8986 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8987   [(match_operand:V8HI 0 "mve_scatter_memory")
8988    (match_operand:V8HI 1 "s_register_operand")
8989    (match_operand:V8HF 2 "s_register_operand")
8990    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8991   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8992 {
8993   rtx ind = XEXP (operands[0], 0);
8994   gcc_assert (REG_P (ind));
8995   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8996                                                        operands[2]));
8997   DONE;
8998 })
8999
9000 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
9001   [(set (mem:BLK (scratch))
9002         (unspec:BLK
9003           [(match_operand:SI 0 "register_operand" "r")
9004            (match_operand:V8HI 1 "s_register_operand" "w")
9005            (match_operand:V8HF 2 "s_register_operand" "w")]
9006           VSTRHQSO_F))]
9007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9008   "vstrh.16\t%q2, [%0, %q1]"
9009   [(set_attr "length" "4")])
9010
9011 ;;
9012 ;; [vstrhq_scatter_offset_p_f]
9013 ;;
9014 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
9015   [(match_operand:V8HI 0 "mve_scatter_memory")
9016    (match_operand:V8HI 1 "s_register_operand")
9017    (match_operand:V8HF 2 "s_register_operand")
9018    (match_operand:HI 3 "vpr_register_operand")
9019    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
9020   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9021 {
9022   rtx ind = XEXP (operands[0], 0);
9023   gcc_assert (REG_P (ind));
9024   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
9025                                                          operands[2],
9026                                                          operands[3]));
9027   DONE;
9028 })
9029
9030 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
9031   [(set (mem:BLK (scratch))
9032         (unspec:BLK
9033           [(match_operand:SI 0 "register_operand" "r")
9034            (match_operand:V8HI 1 "s_register_operand" "w")
9035            (match_operand:V8HF 2 "s_register_operand" "w")
9036            (match_operand:HI 3 "vpr_register_operand" "Up")]
9037           VSTRHQSO_F))]
9038   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9039   "vpst\;vstrht.16\t%q2, [%0, %q1]"
9040   [(set_attr "length" "8")])
9041
9042 ;;
9043 ;; [vstrhq_scatter_shifted_offset_f]
9044 ;;
9045 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
9046   [(match_operand:V8HI 0 "memory_operand" "=Us")
9047    (match_operand:V8HI 1 "s_register_operand" "w")
9048    (match_operand:V8HF 2 "s_register_operand" "w")
9049    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9050   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9051 {
9052   rtx ind = XEXP (operands[0], 0);
9053   gcc_assert (REG_P (ind));
9054   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
9055                                                                operands[2]));
9056   DONE;
9057 })
9058
9059 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
9060   [(set (mem:BLK (scratch))
9061         (unspec:BLK
9062           [(match_operand:SI 0 "register_operand" "r")
9063            (match_operand:V8HI 1 "s_register_operand" "w")
9064            (match_operand:V8HF 2 "s_register_operand" "w")]
9065           VSTRHQSSO_F))]
9066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9067   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
9068   [(set_attr "length" "4")])
9069
9070 ;;
9071 ;; [vstrhq_scatter_shifted_offset_p_f]
9072 ;;
9073 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9074   [(match_operand:V8HI 0 "memory_operand" "=Us")
9075    (match_operand:V8HI 1 "s_register_operand" "w")
9076    (match_operand:V8HF 2 "s_register_operand" "w")
9077    (match_operand:HI 3 "vpr_register_operand" "Up")
9078    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9079   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9080 {
9081   rtx ind = XEXP (operands[0], 0);
9082   gcc_assert (REG_P (ind));
9083   emit_insn (
9084     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
9085                                                         operands[2],
9086                                                         operands[3]));
9087   DONE;
9088 })
9089
9090 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
9091   [(set (mem:BLK (scratch))
9092         (unspec:BLK
9093           [(match_operand:SI 0 "register_operand" "r")
9094            (match_operand:V8HI 1 "s_register_operand" "w")
9095            (match_operand:V8HF 2 "s_register_operand" "w")
9096            (match_operand:HI 3 "vpr_register_operand" "Up")]
9097           VSTRHQSSO_F))]
9098   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9099   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
9100   [(set_attr "length" "8")])
9101
9102 ;;
9103 ;; [vstrwq_scatter_base_f]
9104 ;;
9105 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9106   [(set (mem:BLK (scratch))
9107         (unspec:BLK
9108                 [(match_operand:V4SI 0 "s_register_operand" "w")
9109                  (match_operand:SI 1 "immediate_operand" "i")
9110                  (match_operand:V4SF 2 "s_register_operand" "w")]
9111          VSTRWQSB_F))
9112   ]
9113   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9114 {
9115    rtx ops[3];
9116    ops[0] = operands[0];
9117    ops[1] = operands[1];
9118    ops[2] = operands[2];
9119    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9120    return "";
9121 }
9122   [(set_attr "length" "4")])
9123
9124 ;;
9125 ;; [vstrwq_scatter_base_p_f]
9126 ;;
9127 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9128   [(set (mem:BLK (scratch))
9129         (unspec:BLK
9130                 [(match_operand:V4SI 0 "s_register_operand" "w")
9131                  (match_operand:SI 1 "immediate_operand" "i")
9132                  (match_operand:V4SF 2 "s_register_operand" "w")
9133                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9134          VSTRWQSB_F))
9135   ]
9136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9137 {
9138    rtx ops[3];
9139    ops[0] = operands[0];
9140    ops[1] = operands[1];
9141    ops[2] = operands[2];
9142    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9143    return "";
9144 }
9145   [(set_attr "length" "8")])
9146
9147 ;;
9148 ;; [vstrwq_scatter_offset_f]
9149 ;;
9150 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9151   [(match_operand:V4SI 0 "mve_scatter_memory")
9152    (match_operand:V4SI 1 "s_register_operand")
9153    (match_operand:V4SF 2 "s_register_operand")
9154    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9155   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9156 {
9157   rtx ind = XEXP (operands[0], 0);
9158   gcc_assert (REG_P (ind));
9159   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9160                                                        operands[2]));
9161   DONE;
9162 })
9163
9164 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9165   [(set (mem:BLK (scratch))
9166         (unspec:BLK
9167           [(match_operand:SI 0 "register_operand" "r")
9168            (match_operand:V4SI 1 "s_register_operand" "w")
9169            (match_operand:V4SF 2 "s_register_operand" "w")]
9170           VSTRWQSO_F))]
9171   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9172   "vstrw.32\t%q2, [%0, %q1]"
9173   [(set_attr "length" "4")])
9174
9175 ;;
9176 ;; [vstrwq_scatter_offset_p_f]
9177 ;;
9178 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9179   [(match_operand:V4SI 0 "mve_scatter_memory")
9180    (match_operand:V4SI 1 "s_register_operand")
9181    (match_operand:V4SF 2 "s_register_operand")
9182    (match_operand:HI 3 "vpr_register_operand")
9183    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9184   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9185 {
9186   rtx ind = XEXP (operands[0], 0);
9187   gcc_assert (REG_P (ind));
9188   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9189                                                          operands[2],
9190                                                          operands[3]));
9191   DONE;
9192 })
9193
9194 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9195   [(set (mem:BLK (scratch))
9196         (unspec:BLK
9197           [(match_operand:SI 0 "register_operand" "r")
9198            (match_operand:V4SI 1 "s_register_operand" "w")
9199            (match_operand:V4SF 2 "s_register_operand" "w")
9200            (match_operand:HI 3 "vpr_register_operand" "Up")]
9201           VSTRWQSO_F))]
9202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9203   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9204   [(set_attr "length" "8")])
9205
9206 ;;
9207 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9208 ;;
9209 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9210   [(match_operand:V4SI 0 "mve_scatter_memory")
9211    (match_operand:V4SI 1 "s_register_operand")
9212    (match_operand:V4SI 2 "s_register_operand")
9213    (match_operand:HI 3 "vpr_register_operand")
9214    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9215   "TARGET_HAVE_MVE"
9216 {
9217   rtx ind = XEXP (operands[0], 0);
9218   gcc_assert (REG_P (ind));
9219   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9220                                                               operands[2],
9221                                                               operands[3]));
9222   DONE;
9223 })
9224
9225 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9226   [(set (mem:BLK (scratch))
9227         (unspec:BLK
9228           [(match_operand:SI 0 "register_operand" "r")
9229            (match_operand:V4SI 1 "s_register_operand" "w")
9230            (match_operand:V4SI 2 "s_register_operand" "w")
9231            (match_operand:HI 3 "vpr_register_operand" "Up")]
9232           VSTRWSOQ))]
9233   "TARGET_HAVE_MVE"
9234   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9235   [(set_attr "length" "8")])
9236
9237 ;;
9238 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9239 ;;
9240 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9241   [(match_operand:V4SI 0 "mve_scatter_memory")
9242    (match_operand:V4SI 1 "s_register_operand")
9243    (match_operand:V4SI 2 "s_register_operand")
9244    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9245   "TARGET_HAVE_MVE"
9246 {
9247   rtx ind = XEXP (operands[0], 0);
9248   gcc_assert (REG_P (ind));
9249   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9250                                                             operands[2]));
9251   DONE;
9252 })
9253
9254 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9255   [(set (mem:BLK (scratch))
9256         (unspec:BLK
9257           [(match_operand:SI 0 "register_operand" "r")
9258            (match_operand:V4SI 1 "s_register_operand" "w")
9259            (match_operand:V4SI 2 "s_register_operand" "w")]
9260           VSTRWSOQ))]
9261   "TARGET_HAVE_MVE"
9262   "vstrw.32\t%q2, [%0, %q1]"
9263   [(set_attr "length" "4")])
9264
9265 ;;
9266 ;; [vstrwq_scatter_shifted_offset_f]
9267 ;;
9268 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9269   [(match_operand:V4SI 0 "mve_scatter_memory")
9270    (match_operand:V4SI 1 "s_register_operand")
9271    (match_operand:V4SF 2 "s_register_operand")
9272    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9273   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9274 {
9275   rtx ind = XEXP (operands[0], 0);
9276   gcc_assert (REG_P (ind));
9277   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9278                                                                operands[2]));
9279   DONE;
9280 })
9281
9282 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9283   [(set (mem:BLK (scratch))
9284         (unspec:BLK
9285           [(match_operand:SI 0 "register_operand" "r")
9286            (match_operand:V4SI 1 "s_register_operand" "w")
9287            (match_operand:V4SF 2 "s_register_operand" "w")]
9288          VSTRWQSSO_F))]
9289   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9290   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9291   [(set_attr "length" "8")])
9292
9293 ;;
9294 ;; [vstrwq_scatter_shifted_offset_p_f]
9295 ;;
9296 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9297   [(match_operand:V4SI 0 "mve_scatter_memory")
9298    (match_operand:V4SI 1 "s_register_operand")
9299    (match_operand:V4SF 2 "s_register_operand")
9300    (match_operand:HI 3 "vpr_register_operand")
9301    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9303 {
9304   rtx ind = XEXP (operands[0], 0);
9305   gcc_assert (REG_P (ind));
9306   emit_insn (
9307     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9308                                                         operands[2],
9309                                                         operands[3]));
9310   DONE;
9311 })
9312
9313 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9314   [(set (mem:BLK (scratch))
9315         (unspec:BLK
9316           [(match_operand:SI 0 "register_operand" "r")
9317            (match_operand:V4SI 1 "s_register_operand" "w")
9318            (match_operand:V4SF 2 "s_register_operand" "w")
9319            (match_operand:HI 3 "vpr_register_operand" "Up")]
9320           VSTRWQSSO_F))]
9321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9322   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9323   [(set_attr "length" "8")])
9324
9325 ;;
9326 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9327 ;;
9328 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9329   [(match_operand:V4SI 0 "mve_scatter_memory")
9330    (match_operand:V4SI 1 "s_register_operand")
9331    (match_operand:V4SI 2 "s_register_operand")
9332    (match_operand:HI 3 "vpr_register_operand")
9333    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9334   "TARGET_HAVE_MVE"
9335 {
9336   rtx ind = XEXP (operands[0], 0);
9337   gcc_assert (REG_P (ind));
9338   emit_insn (
9339     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9340                                                              operands[2],
9341                                                              operands[3]));
9342   DONE;
9343 })
9344
9345 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9346   [(set (mem:BLK (scratch))
9347         (unspec:BLK
9348           [(match_operand:SI 0 "register_operand" "r")
9349            (match_operand:V4SI 1 "s_register_operand" "w")
9350            (match_operand:V4SI 2 "s_register_operand" "w")
9351            (match_operand:HI 3 "vpr_register_operand" "Up")]
9352           VSTRWSSOQ))]
9353   "TARGET_HAVE_MVE"
9354   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9355   [(set_attr "length" "8")])
9356
9357 ;;
9358 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9359 ;;
9360 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9361   [(match_operand:V4SI 0 "mve_scatter_memory")
9362    (match_operand:V4SI 1 "s_register_operand")
9363    (match_operand:V4SI 2 "s_register_operand")
9364    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9365   "TARGET_HAVE_MVE"
9366 {
9367   rtx ind = XEXP (operands[0], 0);
9368   gcc_assert (REG_P (ind));
9369   emit_insn (
9370     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9371                                                            operands[2]));
9372   DONE;
9373 })
9374
9375 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9376   [(set (mem:BLK (scratch))
9377         (unspec:BLK
9378           [(match_operand:SI 0 "register_operand" "r")
9379            (match_operand:V4SI 1 "s_register_operand" "w")
9380            (match_operand:V4SI 2 "s_register_operand" "w")]
9381           VSTRWSSOQ))]
9382   "TARGET_HAVE_MVE"
9383   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9384   [(set_attr "length" "4")])
9385
9386 ;;
9387 ;; [vaddq_s, vaddq_u])
9388 ;;
9389 (define_insn "mve_vaddq<mode>"
9390   [
9391    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9392         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9393                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9394   ]
9395   "TARGET_HAVE_MVE"
9396   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9397   [(set_attr "type" "mve_move")
9398 ])
9399
9400 ;;
9401 ;; [vaddq_f])
9402 ;;
9403 (define_insn "mve_vaddq_f<mode>"
9404   [
9405    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9406         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9407                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9408   ]
9409   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9410   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9411   [(set_attr "type" "mve_move")
9412 ])
9413
9414 ;;
9415 ;; [vidupq_n_u])
9416 ;;
9417 (define_expand "mve_vidupq_n_u<mode>"
9418  [(match_operand:MVE_2 0 "s_register_operand")
9419   (match_operand:SI 1 "s_register_operand")
9420   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9421  "TARGET_HAVE_MVE"
9422 {
9423   rtx temp = gen_reg_rtx (SImode);
9424   emit_move_insn (temp, operands[1]);
9425   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9426   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9427                                           operands[2], inc));
9428   DONE;
9429 })
9430
9431 ;;
9432 ;; [vidupq_u_insn])
9433 ;;
9434 (define_insn "mve_vidupq_u<mode>_insn"
9435  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9436        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9437                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9438          VIDUPQ))
9439   (set (match_operand:SI 1 "s_register_operand" "=Te")
9440        (plus:SI (match_dup 2)
9441                 (match_operand:SI 4 "immediate_operand" "i")))]
9442  "TARGET_HAVE_MVE"
9443  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9444
9445 ;;
9446 ;; [vidupq_m_n_u])
9447 ;;
9448 (define_expand "mve_vidupq_m_n_u<mode>"
9449   [(match_operand:MVE_2 0 "s_register_operand")
9450    (match_operand:MVE_2 1 "s_register_operand")
9451    (match_operand:SI 2 "s_register_operand")
9452    (match_operand:SI 3 "mve_imm_selective_upto_8")
9453    (match_operand:HI 4 "vpr_register_operand")]
9454   "TARGET_HAVE_MVE"
9455 {
9456   rtx temp = gen_reg_rtx (SImode);
9457   emit_move_insn (temp, operands[2]);
9458   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9459   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9460                                              operands[2], operands[3],
9461                                              operands[4], inc));
9462   DONE;
9463 })
9464
9465 ;;
9466 ;; [vidupq_m_wb_u_insn])
9467 ;;
9468 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9469  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9470        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9471                       (match_operand:SI 3 "s_register_operand" "2")
9472                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9473                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9474         VIDUPQ_M))
9475   (set (match_operand:SI 2 "s_register_operand" "=Te")
9476        (plus:SI (match_dup 3)
9477                 (match_operand:SI 6 "immediate_operand" "i")))]
9478  "TARGET_HAVE_MVE"
9479  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9480  [(set_attr "length""8")])
9481
9482 ;;
9483 ;; [vddupq_n_u])
9484 ;;
9485 (define_expand "mve_vddupq_n_u<mode>"
9486  [(match_operand:MVE_2 0 "s_register_operand")
9487   (match_operand:SI 1 "s_register_operand")
9488   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9489  "TARGET_HAVE_MVE"
9490 {
9491   rtx temp = gen_reg_rtx (SImode);
9492   emit_move_insn (temp, operands[1]);
9493   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9494   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9495                                           operands[2], inc));
9496   DONE;
9497 })
9498
9499 ;;
9500 ;; [vddupq_u_insn])
9501 ;;
9502 (define_insn "mve_vddupq_u<mode>_insn"
9503  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9504        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9505                       (match_operand:SI 3 "immediate_operand" "i")]
9506         VDDUPQ))
9507   (set (match_operand:SI 1 "s_register_operand" "=Te")
9508        (minus:SI (match_dup 2)
9509                  (match_operand:SI 4 "immediate_operand" "i")))]
9510  "TARGET_HAVE_MVE"
9511  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9512
9513 ;;
9514 ;; [vddupq_m_n_u])
9515 ;;
9516 (define_expand "mve_vddupq_m_n_u<mode>"
9517   [(match_operand:MVE_2 0 "s_register_operand")
9518    (match_operand:MVE_2 1 "s_register_operand")
9519    (match_operand:SI 2 "s_register_operand")
9520    (match_operand:SI 3 "mve_imm_selective_upto_8")
9521    (match_operand:HI 4 "vpr_register_operand")]
9522   "TARGET_HAVE_MVE"
9523 {
9524   rtx temp = gen_reg_rtx (SImode);
9525   emit_move_insn (temp, operands[2]);
9526   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9527   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9528                                              operands[2], operands[3],
9529                                              operands[4], inc));
9530   DONE;
9531 })
9532
9533 ;;
9534 ;; [vddupq_m_wb_u_insn])
9535 ;;
9536 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9537  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9538        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9539                       (match_operand:SI 3 "s_register_operand" "2")
9540                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9541                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9542         VDDUPQ_M))
9543   (set (match_operand:SI 2 "s_register_operand" "=Te")
9544        (minus:SI (match_dup 3)
9545                  (match_operand:SI 6 "immediate_operand" "i")))]
9546  "TARGET_HAVE_MVE"
9547  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9548  [(set_attr "length""8")])
9549
9550 ;;
9551 ;; [vdwdupq_n_u])
9552 ;;
9553 (define_expand "mve_vdwdupq_n_u<mode>"
9554  [(match_operand:MVE_2 0 "s_register_operand")
9555   (match_operand:SI 1 "s_register_operand")
9556   (match_operand:DI 2 "s_register_operand")
9557   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9558  "TARGET_HAVE_MVE"
9559 {
9560   rtx ignore_wb = gen_reg_rtx (SImode);
9561   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9562                                               operands[1], operands[2],
9563                                               operands[3]));
9564   DONE;
9565 })
9566
9567 ;;
9568 ;; [vdwdupq_wb_u])
9569 ;;
9570 (define_expand "mve_vdwdupq_wb_u<mode>"
9571  [(match_operand:SI 0 "s_register_operand")
9572   (match_operand:SI 1 "s_register_operand")
9573   (match_operand:DI 2 "s_register_operand")
9574   (match_operand:SI 3 "mve_imm_selective_upto_8")
9575   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9576  "TARGET_HAVE_MVE"
9577 {
9578   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9579   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9580                                               operands[1], operands[2],
9581                                               operands[3]));
9582   DONE;
9583 })
9584
9585 ;;
9586 ;; [vdwdupq_wb_u_insn])
9587 ;;
9588 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9589   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9590         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9591                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9592                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9593          VDWDUPQ))
9594    (set (match_operand:SI 1 "s_register_operand" "=Te")
9595         (unspec:SI [(match_dup 2)
9596                     (subreg:SI (match_dup 3) 4)
9597                     (match_dup 4)]
9598          VDWDUPQ))]
9599   "TARGET_HAVE_MVE"
9600   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9601 )
9602
9603 ;;
9604 ;; [vdwdupq_m_n_u])
9605 ;;
9606 (define_expand "mve_vdwdupq_m_n_u<mode>"
9607  [(match_operand:MVE_2 0 "s_register_operand")
9608   (match_operand:MVE_2 1 "s_register_operand")
9609   (match_operand:SI 2 "s_register_operand")
9610   (match_operand:DI 3 "s_register_operand")
9611   (match_operand:SI 4 "mve_imm_selective_upto_8")
9612   (match_operand:HI 5 "vpr_register_operand")]
9613  "TARGET_HAVE_MVE"
9614 {
9615   rtx ignore_wb = gen_reg_rtx (SImode);
9616   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9617                                                 operands[1], operands[2],
9618                                                 operands[3], operands[4],
9619                                                 operands[5]));
9620   DONE;
9621 })
9622
9623 ;;
9624 ;; [vdwdupq_m_wb_u])
9625 ;;
9626 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9627  [(match_operand:SI 0 "s_register_operand")
9628   (match_operand:MVE_2 1 "s_register_operand")
9629   (match_operand:SI 2 "s_register_operand")
9630   (match_operand:DI 3 "s_register_operand")
9631   (match_operand:SI 4 "mve_imm_selective_upto_8")
9632   (match_operand:HI 5 "vpr_register_operand")]
9633  "TARGET_HAVE_MVE"
9634 {
9635   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9636   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9637                                                 operands[1], operands[2],
9638                                                 operands[3], operands[4],
9639                                                 operands[5]));
9640   DONE;
9641 })
9642
9643 ;;
9644 ;; [vdwdupq_m_wb_u_insn])
9645 ;;
9646 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9647   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9648         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9649                        (match_operand:SI 3 "s_register_operand" "1")
9650                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9651                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9652                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9653          VDWDUPQ_M))
9654    (set (match_operand:SI 1 "s_register_operand" "=Te")
9655         (unspec:SI [(match_dup 2)
9656                     (match_dup 3)
9657                     (subreg:SI (match_dup 4) 4)
9658                     (match_dup 5)
9659                     (match_dup 6)]
9660          VDWDUPQ_M))
9661   ]
9662   "TARGET_HAVE_MVE"
9663   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9664   [(set_attr "type" "mve_move")
9665    (set_attr "length""8")])
9666
9667 ;;
9668 ;; [viwdupq_n_u])
9669 ;;
9670 (define_expand "mve_viwdupq_n_u<mode>"
9671  [(match_operand:MVE_2 0 "s_register_operand")
9672   (match_operand:SI 1 "s_register_operand")
9673   (match_operand:DI 2 "s_register_operand")
9674   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9675  "TARGET_HAVE_MVE"
9676 {
9677   rtx ignore_wb = gen_reg_rtx (SImode);
9678   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9679                                               operands[1], operands[2],
9680                                               operands[3]));
9681   DONE;
9682 })
9683
9684 ;;
9685 ;; [viwdupq_wb_u])
9686 ;;
9687 (define_expand "mve_viwdupq_wb_u<mode>"
9688  [(match_operand:SI 0 "s_register_operand")
9689   (match_operand:SI 1 "s_register_operand")
9690   (match_operand:DI 2 "s_register_operand")
9691   (match_operand:SI 3 "mve_imm_selective_upto_8")
9692   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9693  "TARGET_HAVE_MVE"
9694 {
9695   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9696   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9697                                               operands[1], operands[2],
9698                                               operands[3]));
9699   DONE;
9700 })
9701
9702 ;;
9703 ;; [viwdupq_wb_u_insn])
9704 ;;
9705 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9706   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9707         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9708                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9709                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9710          VIWDUPQ))
9711    (set (match_operand:SI 1 "s_register_operand" "=Te")
9712         (unspec:SI [(match_dup 2)
9713                     (subreg:SI (match_dup 3) 4)
9714                     (match_dup 4)]
9715          VIWDUPQ))]
9716   "TARGET_HAVE_MVE"
9717   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9718 )
9719
9720 ;;
9721 ;; [viwdupq_m_n_u])
9722 ;;
9723 (define_expand "mve_viwdupq_m_n_u<mode>"
9724  [(match_operand:MVE_2 0 "s_register_operand")
9725   (match_operand:MVE_2 1 "s_register_operand")
9726   (match_operand:SI 2 "s_register_operand")
9727   (match_operand:DI 3 "s_register_operand")
9728   (match_operand:SI 4 "mve_imm_selective_upto_8")
9729   (match_operand:HI 5 "vpr_register_operand")]
9730  "TARGET_HAVE_MVE"
9731 {
9732   rtx ignore_wb = gen_reg_rtx (SImode);
9733   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9734                                                 operands[1], operands[2],
9735                                                 operands[3], operands[4],
9736                                                 operands[5]));
9737   DONE;
9738 })
9739
9740 ;;
9741 ;; [viwdupq_m_wb_u])
9742 ;;
9743 (define_expand "mve_viwdupq_m_wb_u<mode>"
9744  [(match_operand:SI 0 "s_register_operand")
9745   (match_operand:MVE_2 1 "s_register_operand")
9746   (match_operand:SI 2 "s_register_operand")
9747   (match_operand:DI 3 "s_register_operand")
9748   (match_operand:SI 4 "mve_imm_selective_upto_8")
9749   (match_operand:HI 5 "vpr_register_operand")]
9750  "TARGET_HAVE_MVE"
9751 {
9752   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9753   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9754                                                 operands[1], operands[2],
9755                                                 operands[3], operands[4],
9756                                                 operands[5]));
9757   DONE;
9758 })
9759
9760 ;;
9761 ;; [viwdupq_m_wb_u_insn])
9762 ;;
9763 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9764   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9765         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9766                        (match_operand:SI 3 "s_register_operand" "1")
9767                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9768                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9769                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9770          VIWDUPQ_M))
9771    (set (match_operand:SI 1 "s_register_operand" "=Te")
9772         (unspec:SI [(match_dup 2)
9773                     (match_dup 3)
9774                     (subreg:SI (match_dup 4) 4)
9775                     (match_dup 5)
9776                     (match_dup 6)]
9777          VIWDUPQ_M))
9778   ]
9779   "TARGET_HAVE_MVE"
9780   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9781   [(set_attr "type" "mve_move")
9782    (set_attr "length""8")])
9783
9784 ;;
9785 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9786 ;;
9787 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9788   [(set (mem:BLK (scratch))
9789         (unspec:BLK
9790                 [(match_operand:V4SI 1 "s_register_operand" "0")
9791                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9792                  (match_operand:V4SI 3 "s_register_operand" "w")]
9793          VSTRWSBWBQ))
9794    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9795         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9796          VSTRWSBWBQ))
9797   ]
9798   "TARGET_HAVE_MVE"
9799 {
9800    rtx ops[3];
9801    ops[0] = operands[1];
9802    ops[1] = operands[2];
9803    ops[2] = operands[3];
9804    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9805    return "";
9806 }
9807   [(set_attr "length" "4")])
9808
9809 ;;
9810 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9811 ;;
9812 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9813  [(set (mem:BLK (scratch))
9814        (unspec:BLK
9815                 [(match_operand:V4SI 1 "s_register_operand" "0")
9816                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9817                  (match_operand:V4SI 3 "s_register_operand" "w")
9818                  (match_operand:HI 4 "vpr_register_operand")]
9819         VSTRWSBWBQ))
9820    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9821         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9822          VSTRWSBWBQ))
9823   ]
9824   "TARGET_HAVE_MVE"
9825 {
9826    rtx ops[3];
9827    ops[0] = operands[1];
9828    ops[1] = operands[2];
9829    ops[2] = operands[3];
9830    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9831    return "";
9832 }
9833   [(set_attr "length" "8")])
9834
9835 ;;
9836 ;; [vstrwq_scatter_base_wb_f]
9837 ;;
9838 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9839  [(set (mem:BLK (scratch))
9840        (unspec:BLK
9841                 [(match_operand:V4SI 1 "s_register_operand" "0")
9842                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9843                  (match_operand:V4SF 3 "s_register_operand" "w")]
9844          VSTRWQSBWB_F))
9845    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9846         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9847          VSTRWQSBWB_F))
9848   ]
9849   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9850 {
9851    rtx ops[3];
9852    ops[0] = operands[1];
9853    ops[1] = operands[2];
9854    ops[2] = operands[3];
9855    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9856    return "";
9857 }
9858   [(set_attr "length" "4")])
9859
9860 ;;
9861 ;; [vstrwq_scatter_base_wb_p_f]
9862 ;;
9863 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9864  [(set (mem:BLK (scratch))
9865        (unspec:BLK
9866                 [(match_operand:V4SI 1 "s_register_operand" "0")
9867                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9868                  (match_operand:V4SF 3 "s_register_operand" "w")
9869                  (match_operand:HI 4 "vpr_register_operand")]
9870         VSTRWQSBWB_F))
9871    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9872         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9873          VSTRWQSBWB_F))
9874   ]
9875   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9876 {
9877    rtx ops[3];
9878    ops[0] = operands[1];
9879    ops[1] = operands[2];
9880    ops[2] = operands[3];
9881    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9882    return "";
9883 }
9884   [(set_attr "length" "8")])
9885
9886 ;;
9887 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9888 ;;
9889 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9890   [(set (mem:BLK (scratch))
9891         (unspec:BLK
9892                 [(match_operand:V2DI 1 "s_register_operand" "0")
9893                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9894                  (match_operand:V2DI 3 "s_register_operand" "w")]
9895          VSTRDSBWBQ))
9896    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9897         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9898          VSTRDSBWBQ))
9899   ]
9900   "TARGET_HAVE_MVE"
9901 {
9902    rtx ops[3];
9903    ops[0] = operands[1];
9904    ops[1] = operands[2];
9905    ops[2] = operands[3];
9906    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9907    return "";
9908 }
9909   [(set_attr "length" "4")])
9910
9911 ;;
9912 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9913 ;;
9914 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9915   [(set (mem:BLK (scratch))
9916         (unspec:BLK
9917                 [(match_operand:V2DI 1 "s_register_operand" "0")
9918                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9919                  (match_operand:V2DI 3 "s_register_operand" "w")
9920                  (match_operand:HI 4 "vpr_register_operand")]
9921          VSTRDSBWBQ))
9922    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9923         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9924          VSTRDSBWBQ))
9925   ]
9926   "TARGET_HAVE_MVE"
9927 {
9928    rtx ops[3];
9929    ops[0] = operands[1];
9930    ops[1] = operands[2];
9931    ops[2] = operands[3];
9932    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9933    return "";
9934 }
9935   [(set_attr "length" "8")])
9936
9937 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9938   [(match_operand:V4SI 0 "s_register_operand")
9939    (match_operand:V4SI 1 "s_register_operand")
9940    (match_operand:SI 2 "mve_vldrd_immediate")
9941    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9942   "TARGET_HAVE_MVE"
9943 {
9944   rtx ignore_result = gen_reg_rtx (V4SImode);
9945   emit_insn (
9946   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9947                                                  operands[1], operands[2]));
9948   DONE;
9949 })
9950
9951 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9952   [(match_operand:V4SI 0 "s_register_operand")
9953    (match_operand:V4SI 1 "s_register_operand")
9954    (match_operand:SI 2 "mve_vldrd_immediate")
9955    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9956   "TARGET_HAVE_MVE"
9957 {
9958   rtx ignore_wb = gen_reg_rtx (V4SImode);
9959   emit_insn (
9960   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9961                                                  operands[1], operands[2]));
9962   DONE;
9963 })
9964
9965 ;;
9966 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9967 ;;
9968 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9969   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9970         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9971                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9972                       (mem:BLK (scratch))]
9973          VLDRWGBWBQ))
9974    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9975         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9976          VLDRWGBWBQ))
9977   ]
9978   "TARGET_HAVE_MVE"
9979 {
9980    rtx ops[3];
9981    ops[0] = operands[0];
9982    ops[1] = operands[2];
9983    ops[2] = operands[3];
9984    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9985    return "";
9986 }
9987   [(set_attr "length" "4")])
9988
9989 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9990   [(match_operand:V4SI 0 "s_register_operand")
9991    (match_operand:V4SI 1 "s_register_operand")
9992    (match_operand:SI 2 "mve_vldrd_immediate")
9993    (match_operand:HI 3 "vpr_register_operand")
9994    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9995   "TARGET_HAVE_MVE"
9996 {
9997   rtx ignore_result = gen_reg_rtx (V4SImode);
9998   emit_insn (
9999   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
10000                                                    operands[1], operands[2],
10001                                                    operands[3]));
10002   DONE;
10003 })
10004 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
10005   [(match_operand:V4SI 0 "s_register_operand")
10006    (match_operand:V4SI 1 "s_register_operand")
10007    (match_operand:SI 2 "mve_vldrd_immediate")
10008    (match_operand:HI 3 "vpr_register_operand")
10009    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10010   "TARGET_HAVE_MVE"
10011 {
10012   rtx ignore_wb = gen_reg_rtx (V4SImode);
10013   emit_insn (
10014   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10015                                                    operands[1], operands[2],
10016                                                    operands[3]));
10017   DONE;
10018 })
10019
10020 ;;
10021 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10022 ;;
10023 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10024   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10025         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10026                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10027                       (match_operand:HI 4 "vpr_register_operand" "Up")
10028                       (mem:BLK (scratch))]
10029          VLDRWGBWBQ))
10030    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10031         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10032          VLDRWGBWBQ))
10033   ]
10034   "TARGET_HAVE_MVE"
10035 {
10036    rtx ops[3];
10037    ops[0] = operands[0];
10038    ops[1] = operands[2];
10039    ops[2] = operands[3];
10040    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10041    return "";
10042 }
10043   [(set_attr "length" "8")])
10044
10045 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10046   [(match_operand:V4SI 0 "s_register_operand")
10047    (match_operand:V4SI 1 "s_register_operand")
10048    (match_operand:SI 2 "mve_vldrd_immediate")
10049    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10050   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10051 {
10052   rtx ignore_result = gen_reg_rtx (V4SFmode);
10053   emit_insn (
10054   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10055                                             operands[1], operands[2]));
10056   DONE;
10057 })
10058
10059 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10060   [(match_operand:V4SF 0 "s_register_operand")
10061    (match_operand:V4SI 1 "s_register_operand")
10062    (match_operand:SI 2 "mve_vldrd_immediate")
10063    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10064   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10065 {
10066   rtx ignore_wb = gen_reg_rtx (V4SImode);
10067   emit_insn (
10068   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10069                                             operands[1], operands[2]));
10070   DONE;
10071 })
10072
10073 ;;
10074 ;; [vldrwq_gather_base_wb_f]
10075 ;;
10076 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10077   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10078         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10079                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10080                       (mem:BLK (scratch))]
10081          VLDRWQGBWB_F))
10082    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10083         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10084          VLDRWQGBWB_F))
10085   ]
10086   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10087 {
10088    rtx ops[3];
10089    ops[0] = operands[0];
10090    ops[1] = operands[2];
10091    ops[2] = operands[3];
10092    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10093    return "";
10094 }
10095   [(set_attr "length" "4")])
10096
10097 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10098   [(match_operand:V4SI 0 "s_register_operand")
10099    (match_operand:V4SI 1 "s_register_operand")
10100    (match_operand:SI 2 "mve_vldrd_immediate")
10101    (match_operand:HI 3 "vpr_register_operand")
10102    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10103   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10104 {
10105   rtx ignore_result = gen_reg_rtx (V4SFmode);
10106   emit_insn (
10107   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10108                                               operands[1], operands[2],
10109                                               operands[3]));
10110   DONE;
10111 })
10112
10113 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10114   [(match_operand:V4SF 0 "s_register_operand")
10115    (match_operand:V4SI 1 "s_register_operand")
10116    (match_operand:SI 2 "mve_vldrd_immediate")
10117    (match_operand:HI 3 "vpr_register_operand")
10118    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10119   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10120 {
10121   rtx ignore_wb = gen_reg_rtx (V4SImode);
10122   emit_insn (
10123   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10124                                               operands[1], operands[2],
10125                                               operands[3]));
10126   DONE;
10127 })
10128
10129 ;;
10130 ;; [vldrwq_gather_base_wb_z_f]
10131 ;;
10132 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10133   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10134         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10135                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10136                       (match_operand:HI 4 "vpr_register_operand" "Up")
10137                       (mem:BLK (scratch))]
10138          VLDRWQGBWB_F))
10139    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10140         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10141          VLDRWQGBWB_F))
10142   ]
10143   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10144 {
10145    rtx ops[3];
10146    ops[0] = operands[0];
10147    ops[1] = operands[2];
10148    ops[2] = operands[3];
10149    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10150    return "";
10151 }
10152   [(set_attr "length" "8")])
10153
10154 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10155   [(match_operand:V2DI 0 "s_register_operand")
10156    (match_operand:V2DI 1 "s_register_operand")
10157    (match_operand:SI 2 "mve_vldrd_immediate")
10158    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10159   "TARGET_HAVE_MVE"
10160 {
10161   rtx ignore_result = gen_reg_rtx (V2DImode);
10162   emit_insn (
10163   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10164                                                  operands[1], operands[2]));
10165   DONE;
10166 })
10167
10168 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10169   [(match_operand:V2DI 0 "s_register_operand")
10170    (match_operand:V2DI 1 "s_register_operand")
10171    (match_operand:SI 2 "mve_vldrd_immediate")
10172    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10173   "TARGET_HAVE_MVE"
10174 {
10175   rtx ignore_wb = gen_reg_rtx (V2DImode);
10176   emit_insn (
10177   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10178                                                  operands[1], operands[2]));
10179   DONE;
10180 })
10181
10182
10183 ;;
10184 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10185 ;;
10186 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10187   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10188         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10189                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10190                       (mem:BLK (scratch))]
10191          VLDRDGBWBQ))
10192    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10193         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10194          VLDRDGBWBQ))
10195   ]
10196   "TARGET_HAVE_MVE"
10197 {
10198    rtx ops[3];
10199    ops[0] = operands[0];
10200    ops[1] = operands[2];
10201    ops[2] = operands[3];
10202    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10203    return "";
10204 }
10205   [(set_attr "length" "4")])
10206
10207 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10208   [(match_operand:V2DI 0 "s_register_operand")
10209    (match_operand:V2DI 1 "s_register_operand")
10210    (match_operand:SI 2 "mve_vldrd_immediate")
10211    (match_operand:HI 3 "vpr_register_operand")
10212    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10213   "TARGET_HAVE_MVE"
10214 {
10215   rtx ignore_result = gen_reg_rtx (V2DImode);
10216   emit_insn (
10217   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10218                                                    operands[1], operands[2],
10219                                                    operands[3]));
10220   DONE;
10221 })
10222
10223 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10224   [(match_operand:V2DI 0 "s_register_operand")
10225    (match_operand:V2DI 1 "s_register_operand")
10226    (match_operand:SI 2 "mve_vldrd_immediate")
10227    (match_operand:HI 3 "vpr_register_operand")
10228    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10229   "TARGET_HAVE_MVE"
10230 {
10231   rtx ignore_wb = gen_reg_rtx (V2DImode);
10232   emit_insn (
10233   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10234                                                    operands[1], operands[2],
10235                                                    operands[3]));
10236   DONE;
10237 })
10238
10239 (define_insn "get_fpscr_nzcvqc"
10240  [(set (match_operand:SI 0 "register_operand" "=r")
10241    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10242  "TARGET_HAVE_MVE"
10243  "vmrs\\t%0, FPSCR_nzcvqc"
10244   [(set_attr "type" "mve_move")])
10245
10246 (define_insn "set_fpscr_nzcvqc"
10247  [(set (reg:SI VFPCC_REGNUM)
10248    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10249     VUNSPEC_SET_FPSCR_NZCVQC))]
10250  "TARGET_HAVE_MVE"
10251  "vmsr\\tFPSCR_nzcvqc, %0"
10252   [(set_attr "type" "mve_move")])
10253
10254 ;;
10255 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10256 ;;
10257 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10258   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10259         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10260                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10261                       (match_operand:HI 4 "vpr_register_operand" "Up")
10262                       (mem:BLK (scratch))]
10263          VLDRDGBWBQ))
10264    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10265         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10266          VLDRDGBWBQ))
10267   ]
10268   "TARGET_HAVE_MVE"
10269 {
10270    rtx ops[3];
10271    ops[0] = operands[0];
10272    ops[1] = operands[2];
10273    ops[2] = operands[3];
10274    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10275    return "";
10276 }
10277   [(set_attr "length" "8")])
10278 ;;
10279 ;; [vadciq_m_s, vadciq_m_u])
10280 ;;
10281 (define_insn "mve_vadciq_m_<supf>v4si"
10282   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10283         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10284                       (match_operand:V4SI 2 "s_register_operand" "w")
10285                       (match_operand:V4SI 3 "s_register_operand" "w")
10286                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10287          VADCIQ_M))
10288    (set (reg:SI VFPCC_REGNUM)
10289         (unspec:SI [(const_int 0)]
10290          VADCIQ_M))
10291   ]
10292   "TARGET_HAVE_MVE"
10293   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10294   [(set_attr "type" "mve_move")
10295    (set_attr "length" "8")])
10296
10297 ;;
10298 ;; [vadciq_u, vadciq_s])
10299 ;;
10300 (define_insn "mve_vadciq_<supf>v4si"
10301   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10302         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10303                       (match_operand:V4SI 2 "s_register_operand" "w")]
10304          VADCIQ))
10305    (set (reg:SI VFPCC_REGNUM)
10306         (unspec:SI [(const_int 0)]
10307          VADCIQ))
10308   ]
10309   "TARGET_HAVE_MVE"
10310   "vadci.i32\t%q0, %q1, %q2"
10311   [(set_attr "type" "mve_move")
10312    (set_attr "length" "4")])
10313
10314 ;;
10315 ;; [vadcq_m_s, vadcq_m_u])
10316 ;;
10317 (define_insn "mve_vadcq_m_<supf>v4si"
10318   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10319         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10320                       (match_operand:V4SI 2 "s_register_operand" "w")
10321                       (match_operand:V4SI 3 "s_register_operand" "w")
10322                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10323          VADCQ_M))
10324    (set (reg:SI VFPCC_REGNUM)
10325         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10326          VADCQ_M))
10327   ]
10328   "TARGET_HAVE_MVE"
10329   "vpst\;vadct.i32\t%q0, %q2, %q3"
10330   [(set_attr "type" "mve_move")
10331    (set_attr "length" "8")])
10332
10333 ;;
10334 ;; [vadcq_u, vadcq_s])
10335 ;;
10336 (define_insn "mve_vadcq_<supf>v4si"
10337   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10338         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10339                        (match_operand:V4SI 2 "s_register_operand" "w")]
10340          VADCQ))
10341    (set (reg:SI VFPCC_REGNUM)
10342         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10343          VADCQ))
10344   ]
10345   "TARGET_HAVE_MVE"
10346   "vadc.i32\t%q0, %q1, %q2"
10347   [(set_attr "type" "mve_move")
10348    (set_attr "length" "4")
10349    (set_attr "conds" "set")])
10350
10351 ;;
10352 ;; [vsbciq_m_u, vsbciq_m_s])
10353 ;;
10354 (define_insn "mve_vsbciq_m_<supf>v4si"
10355   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10356         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10357                       (match_operand:V4SI 2 "s_register_operand" "w")
10358                       (match_operand:V4SI 3 "s_register_operand" "w")
10359                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10360          VSBCIQ_M))
10361    (set (reg:SI VFPCC_REGNUM)
10362         (unspec:SI [(const_int 0)]
10363          VSBCIQ_M))
10364   ]
10365   "TARGET_HAVE_MVE"
10366   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10367   [(set_attr "type" "mve_move")
10368    (set_attr "length" "8")])
10369
10370 ;;
10371 ;; [vsbciq_s, vsbciq_u])
10372 ;;
10373 (define_insn "mve_vsbciq_<supf>v4si"
10374   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10375         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10376                       (match_operand:V4SI 2 "s_register_operand" "w")]
10377          VSBCIQ))
10378    (set (reg:SI VFPCC_REGNUM)
10379         (unspec:SI [(const_int 0)]
10380          VSBCIQ))
10381   ]
10382   "TARGET_HAVE_MVE"
10383   "vsbci.i32\t%q0, %q1, %q2"
10384   [(set_attr "type" "mve_move")
10385    (set_attr "length" "4")])
10386
10387 ;;
10388 ;; [vsbcq_m_u, vsbcq_m_s])
10389 ;;
10390 (define_insn "mve_vsbcq_m_<supf>v4si"
10391   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10392         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10393                       (match_operand:V4SI 2 "s_register_operand" "w")
10394                       (match_operand:V4SI 3 "s_register_operand" "w")
10395                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10396          VSBCQ_M))
10397    (set (reg:SI VFPCC_REGNUM)
10398         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10399          VSBCQ_M))
10400   ]
10401   "TARGET_HAVE_MVE"
10402   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10403   [(set_attr "type" "mve_move")
10404    (set_attr "length" "8")])
10405
10406 ;;
10407 ;; [vsbcq_s, vsbcq_u])
10408 ;;
10409 (define_insn "mve_vsbcq_<supf>v4si"
10410   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10411         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10412                       (match_operand:V4SI 2 "s_register_operand" "w")]
10413          VSBCQ))
10414    (set (reg:SI VFPCC_REGNUM)
10415         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10416          VSBCQ))
10417   ]
10418   "TARGET_HAVE_MVE"
10419   "vsbc.i32\t%q0, %q1, %q2"
10420   [(set_attr "type" "mve_move")
10421    (set_attr "length" "4")])
10422
10423 ;;
10424 ;; [vst2q])
10425 ;;
10426 (define_insn "mve_vst2q<mode>"
10427   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10428         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10429                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10430          VST2Q))
10431   ]
10432   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10433    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10434 {
10435    rtx ops[4];
10436    int regno = REGNO (operands[1]);
10437    ops[0] = gen_rtx_REG (TImode, regno);
10438    ops[1] = gen_rtx_REG (TImode, regno + 4);
10439    rtx reg  = operands[0];
10440    while (reg && !REG_P (reg))
10441     reg = XEXP (reg, 0);
10442    gcc_assert (REG_P (reg));
10443    ops[2] = reg;
10444    ops[3] = operands[0];
10445    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10446                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10447    return "";
10448 }
10449   [(set_attr "length" "8")])
10450
10451 ;;
10452 ;; [vld2q])
10453 ;;
10454 (define_insn "mve_vld2q<mode>"
10455   [(set (match_operand:OI 0 "s_register_operand" "=w")
10456         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10457                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10458          VLD2Q))
10459   ]
10460   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10461    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10462 {
10463    rtx ops[4];
10464    int regno = REGNO (operands[0]);
10465    ops[0] = gen_rtx_REG (TImode, regno);
10466    ops[1] = gen_rtx_REG (TImode, regno + 4);
10467    rtx reg  = operands[1];
10468    while (reg && !REG_P (reg))
10469     reg = XEXP (reg, 0);
10470    gcc_assert (REG_P (reg));
10471    ops[2] = reg;
10472    ops[3] = operands[1];
10473    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10474                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10475    return "";
10476 }
10477   [(set_attr "length" "8")])
10478
10479 ;;
10480 ;; [vld4q])
10481 ;;
10482 (define_insn "mve_vld4q<mode>"
10483   [(set (match_operand:XI 0 "s_register_operand" "=w")
10484         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10485                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10486          VLD4Q))
10487   ]
10488   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10489    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10490 {
10491    rtx ops[6];
10492    int regno = REGNO (operands[0]);
10493    ops[0] = gen_rtx_REG (TImode, regno);
10494    ops[1] = gen_rtx_REG (TImode, regno+4);
10495    ops[2] = gen_rtx_REG (TImode, regno+8);
10496    ops[3] = gen_rtx_REG (TImode, regno + 12);
10497    rtx reg  = operands[1];
10498    while (reg && !REG_P (reg))
10499     reg = XEXP (reg, 0);
10500    gcc_assert (REG_P (reg));
10501    ops[4] = reg;
10502    ops[5] = operands[1];
10503    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10504                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10505                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10506                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10507    return "";
10508 }
10509   [(set_attr "length" "16")])
10510 ;;
10511 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10512 ;;
10513 (define_insn "mve_vec_extract<mode><V_elem_l>"
10514  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10515    (vec_select:<V_elem>
10516     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10517     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10518   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10519    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10520 {
10521   if (BYTES_BIG_ENDIAN)
10522     {
10523       int elt = INTVAL (operands[2]);
10524       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10525       operands[2] = GEN_INT (elt);
10526     }
10527   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10528 }
10529  [(set_attr "type" "mve_move")])
10530
10531 (define_insn "mve_vec_extractv2didi"
10532  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10533    (vec_select:DI
10534     (match_operand:V2DI 1 "s_register_operand" "w")
10535     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10536   "TARGET_HAVE_MVE"
10537 {
10538   int elt = INTVAL (operands[2]);
10539   if (BYTES_BIG_ENDIAN)
10540     elt = 1 - elt;
10541
10542   if (elt == 0)
10543    return "vmov\t%Q0, %R0, %e1";
10544   else
10545    return "vmov\t%Q0, %R0, %f1";
10546 }
10547  [(set_attr "type" "mve_move")])
10548
10549 (define_insn "*mve_vec_extract_sext_internal<mode>"
10550  [(set (match_operand:SI 0 "s_register_operand" "=r")
10551    (sign_extend:SI
10552     (vec_select:<V_elem>
10553      (match_operand:MVE_2 1 "s_register_operand" "w")
10554      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10555   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10556    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10557 {
10558   if (BYTES_BIG_ENDIAN)
10559     {
10560       int elt = INTVAL (operands[2]);
10561       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10562       operands[2] = GEN_INT (elt);
10563     }
10564   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10565 }
10566  [(set_attr "type" "mve_move")])
10567
10568 (define_insn "*mve_vec_extract_zext_internal<mode>"
10569  [(set (match_operand:SI 0 "s_register_operand" "=r")
10570    (zero_extend:SI
10571     (vec_select:<V_elem>
10572      (match_operand:MVE_2 1 "s_register_operand" "w")
10573      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10574   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10575    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10576 {
10577   if (BYTES_BIG_ENDIAN)
10578     {
10579       int elt = INTVAL (operands[2]);
10580       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10581       operands[2] = GEN_INT (elt);
10582     }
10583   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10584 }
10585  [(set_attr "type" "mve_move")])
10586
10587 ;;
10588 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10589 ;;
10590 (define_insn "mve_vec_set<mode>_internal"
10591  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10592        (vec_merge:VQ2
10593         (vec_duplicate:VQ2
10594           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10595         (match_operand:VQ2 3 "s_register_operand" "0")
10596         (match_operand:SI 2 "immediate_operand" "i")))]
10597   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10598    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10599 {
10600   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10601   if (BYTES_BIG_ENDIAN)
10602     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10603   operands[2] = GEN_INT (elt);
10604
10605   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10606 }
10607  [(set_attr "type" "mve_move")])
10608
10609 (define_insn "mve_vec_setv2di_internal"
10610  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10611        (vec_merge:V2DI
10612         (vec_duplicate:V2DI
10613           (match_operand:DI 1 "nonimmediate_operand" "r"))
10614         (match_operand:V2DI 3 "s_register_operand" "0")
10615         (match_operand:SI 2 "immediate_operand" "i")))]
10616  "TARGET_HAVE_MVE"
10617 {
10618   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10619   if (BYTES_BIG_ENDIAN)
10620     elt = 1 - elt;
10621
10622   if (elt == 0)
10623    return "vmov\t%e0, %Q1, %R1";
10624   else
10625    return "vmov\t%f0, %J1, %K1";
10626 }
10627  [(set_attr "type" "mve_move")])
10628
10629 ;;
10630 ;; [uqrshll_di]
10631 ;;
10632 (define_insn "mve_uqrshll_sat<supf>_di"
10633   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10634         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10635                     (match_operand:SI 2 "register_operand" "r")]
10636          UQRSHLLQ))]
10637   "TARGET_HAVE_MVE"
10638   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10639   [(set_attr "predicable" "yes")])
10640
10641 ;;
10642 ;; [sqrshrl_di]
10643 ;;
10644 (define_insn "mve_sqrshrl_sat<supf>_di"
10645   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10646         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10647                     (match_operand:SI 2 "register_operand" "r")]
10648          SQRSHRLQ))]
10649   "TARGET_HAVE_MVE"
10650   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10651   [(set_attr "predicable" "yes")])
10652
10653 ;;
10654 ;; [uqrshl_si]
10655 ;;
10656 (define_insn "mve_uqrshl_si"
10657   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10658         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10659                     (match_operand:SI 2 "register_operand" "r")]
10660          UQRSHL))]
10661   "TARGET_HAVE_MVE"
10662   "uqrshl%?\\t%1, %2"
10663   [(set_attr "predicable" "yes")])
10664
10665 ;;
10666 ;; [sqrshr_si]
10667 ;;
10668 (define_insn "mve_sqrshr_si"
10669   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10670         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10671                     (match_operand:SI 2 "register_operand" "r")]
10672          SQRSHR))]
10673   "TARGET_HAVE_MVE"
10674   "sqrshr%?\\t%1, %2"
10675   [(set_attr "predicable" "yes")])
10676
10677 ;;
10678 ;; [uqshll_di]
10679 ;;
10680 (define_insn "mve_uqshll_di"
10681   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10682         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10683                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10684   "TARGET_HAVE_MVE"
10685   "uqshll%?\\t%Q1, %R1, %2"
10686   [(set_attr "predicable" "yes")])
10687
10688 ;;
10689 ;; [urshrl_di]
10690 ;;
10691 (define_insn "mve_urshrl_di"
10692   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10693         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10694                     (match_operand:SI 2 "immediate_operand" "Pg")]
10695          URSHRL))]
10696   "TARGET_HAVE_MVE"
10697   "urshrl%?\\t%Q1, %R1, %2"
10698   [(set_attr "predicable" "yes")])
10699
10700 ;;
10701 ;; [uqshl_si]
10702 ;;
10703 (define_insn "mve_uqshl_si"
10704   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10705         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10706                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10707   "TARGET_HAVE_MVE"
10708   "uqshl%?\\t%1, %2"
10709   [(set_attr "predicable" "yes")])
10710
10711 ;;
10712 ;; [urshr_si]
10713 ;;
10714 (define_insn "mve_urshr_si"
10715   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10716         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10717                     (match_operand:SI 2 "immediate_operand" "Pg")]
10718          URSHR))]
10719   "TARGET_HAVE_MVE"
10720   "urshr%?\\t%1, %2"
10721   [(set_attr "predicable" "yes")])
10722
10723 ;;
10724 ;; [sqshl_si]
10725 ;;
10726 (define_insn "mve_sqshl_si"
10727   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10728         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10729                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10730   "TARGET_HAVE_MVE"
10731   "sqshl%?\\t%1, %2"
10732   [(set_attr "predicable" "yes")])
10733
10734 ;;
10735 ;; [srshr_si]
10736 ;;
10737 (define_insn "mve_srshr_si"
10738   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10739         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10740                     (match_operand:SI 2 "immediate_operand" "Pg")]
10741          SRSHR))]
10742   "TARGET_HAVE_MVE"
10743   "srshr%?\\t%1, %2"
10744   [(set_attr "predicable" "yes")])
10745
10746 ;;
10747 ;; [srshrl_di]
10748 ;;
10749 (define_insn "mve_srshrl_di"
10750   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10751         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10752                     (match_operand:SI 2 "immediate_operand" "Pg")]
10753          SRSHRL))]
10754   "TARGET_HAVE_MVE"
10755   "srshrl%?\\t%Q1, %R1, %2"
10756   [(set_attr "predicable" "yes")])
10757
10758 ;;
10759 ;; [sqshll_di]
10760 ;;
10761 (define_insn "mve_sqshll_di"
10762   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10763         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10764                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10765   "TARGET_HAVE_MVE"
10766   "sqshll%?\\t%Q1, %R1, %2"
10767   [(set_attr "predicable" "yes")])
10768
10769 ;;
10770 ;; [vshlcq_m_u vshlcq_m_s]
10771 ;;
10772 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10773  [(match_operand:MVE_2 0 "s_register_operand")
10774   (match_operand:MVE_2 1 "s_register_operand")
10775   (match_operand:SI 2 "s_register_operand")
10776   (match_operand:SI 3 "mve_imm_32")
10777   (match_operand:HI 4 "vpr_register_operand")
10778   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10779  "TARGET_HAVE_MVE"
10780 {
10781   rtx ignore_wb = gen_reg_rtx (SImode);
10782   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10783                                             operands[2], operands[3],
10784                                             operands[4]));
10785   DONE;
10786 })
10787
10788 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10789  [(match_operand:SI 0 "s_register_operand")
10790   (match_operand:MVE_2 1 "s_register_operand")
10791   (match_operand:SI 2 "s_register_operand")
10792   (match_operand:SI 3 "mve_imm_32")
10793   (match_operand:HI 4 "vpr_register_operand")
10794   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10795  "TARGET_HAVE_MVE"
10796 {
10797   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10798   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10799                                             operands[1], operands[2],
10800                                             operands[3], operands[4]));
10801   DONE;
10802 })
10803
10804 (define_insn "mve_vshlcq_m_<supf><mode>"
10805  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10806        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10807                       (match_operand:SI 3 "s_register_operand" "1")
10808                       (match_operand:SI 4 "mve_imm_32" "Rf")
10809                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10810         VSHLCQ_M))
10811   (set (match_operand:SI  1 "s_register_operand" "=r")
10812        (unspec:SI [(match_dup 2)
10813                    (match_dup 3)
10814                    (match_dup 4)
10815                    (match_dup 5)]
10816         VSHLCQ_M))
10817  ]
10818  "TARGET_HAVE_MVE"
10819  "vpst\;vshlct\t%q0, %1, %4"
10820  [(set_attr "type" "mve_move")
10821   (set_attr "length" "8")])
10822
10823 (define_insn "*mve_vec_duplicate<mode>"
10824  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
10825        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
10826  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
10827  "vdup.<V_sz_elem>\t%q0, %1"
10828  [(set_attr "type" "mve_move")])
10829
10830 ;; CDE instructions on MVE registers.
10831
10832 (define_insn "arm_vcx1qv16qi"
10833   [(set (match_operand:V16QI 0 "register_operand" "=t")
10834         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10835                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10836          UNSPEC_VCDE))]
10837   "TARGET_CDE && TARGET_HAVE_MVE"
10838   "vcx1\\tp%c1, %q0, #%c2"
10839   [(set_attr "type" "coproc")]
10840 )
10841
10842 (define_insn "arm_vcx1qav16qi"
10843   [(set (match_operand:V16QI 0 "register_operand" "=t")
10844         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10845                             (match_operand:V16QI 2 "register_operand" "0")
10846                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10847          UNSPEC_VCDEA))]
10848   "TARGET_CDE && TARGET_HAVE_MVE"
10849   "vcx1a\\tp%c1, %q0, #%c3"
10850   [(set_attr "type" "coproc")]
10851 )
10852
10853 (define_insn "arm_vcx2qv16qi"
10854   [(set (match_operand:V16QI 0 "register_operand" "=t")
10855         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10856                           (match_operand:V16QI 2 "register_operand" "t")
10857                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10858          UNSPEC_VCDE))]
10859   "TARGET_CDE && TARGET_HAVE_MVE"
10860   "vcx2\\tp%c1, %q0, %q2, #%c3"
10861   [(set_attr "type" "coproc")]
10862 )
10863
10864 (define_insn "arm_vcx2qav16qi"
10865   [(set (match_operand:V16QI 0 "register_operand" "=t")
10866         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10867                           (match_operand:V16QI 2 "register_operand" "0")
10868                           (match_operand:V16QI 3 "register_operand" "t")
10869                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10870          UNSPEC_VCDEA))]
10871   "TARGET_CDE && TARGET_HAVE_MVE"
10872   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10873   [(set_attr "type" "coproc")]
10874 )
10875
10876 (define_insn "arm_vcx3qv16qi"
10877   [(set (match_operand:V16QI 0 "register_operand" "=t")
10878         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10879                           (match_operand:V16QI 2 "register_operand" "t")
10880                           (match_operand:V16QI 3 "register_operand" "t")
10881                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10882          UNSPEC_VCDE))]
10883   "TARGET_CDE && TARGET_HAVE_MVE"
10884   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10885   [(set_attr "type" "coproc")]
10886 )
10887
10888 (define_insn "arm_vcx3qav16qi"
10889   [(set (match_operand:V16QI 0 "register_operand" "=t")
10890         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10891                           (match_operand:V16QI 2 "register_operand" "0")
10892                           (match_operand:V16QI 3 "register_operand" "t")
10893                           (match_operand:V16QI 4 "register_operand" "t")
10894                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10895          UNSPEC_VCDEA))]
10896   "TARGET_CDE && TARGET_HAVE_MVE"
10897   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10898   [(set_attr "type" "coproc")]
10899 )
10900
10901 (define_insn "arm_vcx1q<a>_p_v16qi"
10902   [(set (match_operand:V16QI 0 "register_operand" "=t")
10903         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10904                            (match_operand:V16QI 2 "register_operand" "0")
10905                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10906                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10907          CDE_VCX))]
10908   "TARGET_CDE && TARGET_HAVE_MVE"
10909   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10910   [(set_attr "type" "coproc")
10911    (set_attr "length" "8")]
10912 )
10913
10914 (define_insn "arm_vcx2q<a>_p_v16qi"
10915   [(set (match_operand:V16QI 0 "register_operand" "=t")
10916         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10917                           (match_operand:V16QI 2 "register_operand" "0")
10918                           (match_operand:V16QI 3 "register_operand" "t")
10919                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10920                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10921          CDE_VCX))]
10922   "TARGET_CDE && TARGET_HAVE_MVE"
10923   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10924   [(set_attr "type" "coproc")
10925    (set_attr "length" "8")]
10926 )
10927
10928 (define_insn "arm_vcx3q<a>_p_v16qi"
10929   [(set (match_operand:V16QI 0 "register_operand" "=t")
10930         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10931                           (match_operand:V16QI 2 "register_operand" "0")
10932                           (match_operand:V16QI 3 "register_operand" "t")
10933                           (match_operand:V16QI 4 "register_operand" "t")
10934                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10935                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10936          CDE_VCX))]
10937   "TARGET_CDE && TARGET_HAVE_MVE"
10938   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10939   [(set_attr "type" "coproc")
10940    (set_attr "length" "8")]
10941 )