gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Uxi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       rtx ops[2];
  45       int regno = (which_alternative == 7)
  46                   ? REGNO (operands[1]) : REGNO (operands[0]);
  47
  48       ops[0] = operands[0];
  49       ops[1] = operands[1];
  50       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode)
  51         {
  52           if (which_alternative == 7)
  53             {
  54               ops[1] = gen_rtx_REG (DImode, regno);
  55               output_asm_insn ("vstr.64\t%P1, %E0",ops);
  56             }
  57           else
  58             {
  59               ops[0] = gen_rtx_REG (DImode, regno);
  60               output_asm_insn ("vldr.64\t%P0, %E1",ops);
  61             }
  62         }
  63       else if (<MODE>mode == TImode)
  64         {
  65           if (which_alternative == 7)
  66             output_asm_insn ("vstr.64\t%q1, %E0",ops);
  67           else
  68             output_asm_insn ("vldr.64\t%q0, %E1",ops);
  69         }
  70       else
  71         {
  72           if (which_alternative == 7)
  73             {
  74               ops[1] = gen_rtx_REG (TImode, regno);
  75               output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0",ops);
  76             }
  77           else
  78             {
  79               ops[0] = gen_rtx_REG (TImode, regno);
  80               output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1",ops);
  81             }
  82         }
  83       return "";
  84     }
  85   switch (which_alternative)
  86     {
  87     case 0:
  88       return "vmov\t%q0, %q1";
  89     case 1:
  90       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  91     case 2:
  92       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  93     case 5:
  94       return output_move_quad (operands);
  95     case 8:
  96         return output_move_neon (operands);
  97     default:
  98       gcc_unreachable ();
  99       return "";
 100     }
 101 }
 102   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
 103    (set_attr "length" "4,8,8,4,8,8,4,4,4")
 104    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
 105    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
 106
 107 (define_insn "*mve_mov<mode>"
 108   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 109         (vec_duplicate:MVE_types
 110           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 111   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 112 {
 113   if (which_alternative == 0)
 114     return "vdup.<V_sz_elem>\t%q0, %1";
 115   return "vmov.<V_sz_elem>\t%q0, %1";
 116 }
 117   [(set_attr "length" "4,4")
 118    (set_attr "type" "mve_move,mve_move")])
 119
 120 ;;
 121 ;; [vst4q])
 122 ;;
 123 (define_insn "mve_vst4q<mode>"
 124   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 125         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 126                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 127          VST4Q))
 128   ]
 129   "TARGET_HAVE_MVE"
 130 {
 131    rtx ops[6];
 132    int regno = REGNO (operands[1]);
 133    ops[0] = gen_rtx_REG (TImode, regno);
 134    ops[1] = gen_rtx_REG (TImode, regno+4);
 135    ops[2] = gen_rtx_REG (TImode, regno+8);
 136    ops[3] = gen_rtx_REG (TImode, regno+12);
 137    rtx reg  = operands[0];
 138    while (reg && !REG_P (reg))
 139     reg = XEXP (reg, 0);
 140    gcc_assert (REG_P (reg));
 141    ops[4] = reg;
 142    ops[5] = operands[0];
 143    /* Here in first three instructions data is stored to ops[4]'s location but
 144       in the fourth instruction data is stored to operands[0], this is to
 145       support the writeback.  */
 146    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 147                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 148                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 149                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 150    return "";
 151 }
 152   [(set_attr "length" "16")])
 153
 154 ;;
 155 ;; [vrndq_m_f])
 156 ;;
 157 (define_insn "mve_vrndq_m_f<mode>"
 158   [
 159    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 160         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 161                        (match_operand:MVE_0 2 "s_register_operand" "w")
 162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 163          VRNDQ_M_F))
 164   ]
 165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 166   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 167   [(set_attr "type" "mve_move")
 168    (set_attr "length""8")])
 169
 170 ;;
 171 ;; [vrndxq_f])
 172 ;;
 173 (define_insn "mve_vrndxq_f<mode>"
 174   [
 175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 177          VRNDXQ_F))
 178   ]
 179   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 180   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 181   [(set_attr "type" "mve_move")
 182 ])
 183
 184 ;;
 185 ;; [vrndq_f])
 186 ;;
 187 (define_insn "mve_vrndq_f<mode>"
 188   [
 189    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 190         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 191          VRNDQ_F))
 192   ]
 193   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 194   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 195   [(set_attr "type" "mve_move")
 196 ])
 197
 198 ;;
 199 ;; [vrndpq_f])
 200 ;;
 201 (define_insn "mve_vrndpq_f<mode>"
 202   [
 203    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 204         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 205          VRNDPQ_F))
 206   ]
 207   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 208   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 209   [(set_attr "type" "mve_move")
 210 ])
 211
 212 ;;
 213 ;; [vrndnq_f])
 214 ;;
 215 (define_insn "mve_vrndnq_f<mode>"
 216   [
 217    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 218         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 219          VRNDNQ_F))
 220   ]
 221   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 222   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 223   [(set_attr "type" "mve_move")
 224 ])
 225
 226 ;;
 227 ;; [vrndmq_f])
 228 ;;
 229 (define_insn "mve_vrndmq_f<mode>"
 230   [
 231    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 232         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 233          VRNDMQ_F))
 234   ]
 235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 236   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 237   [(set_attr "type" "mve_move")
 238 ])
 239
 240 ;;
 241 ;; [vrndaq_f])
 242 ;;
 243 (define_insn "mve_vrndaq_f<mode>"
 244   [
 245    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 246         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 247          VRNDAQ_F))
 248   ]
 249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 250   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 251   [(set_attr "type" "mve_move")
 252 ])
 253
 254 ;;
 255 ;; [vrev64q_f])
 256 ;;
 257 (define_insn "mve_vrev64q_f<mode>"
 258   [
 259    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 260         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 261          VREV64Q_F))
 262   ]
 263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 264   "vrev64.%#<V_sz_elem> %q0, %q1"
 265   [(set_attr "type" "mve_move")
 266 ])
 267
 268 ;;
 269 ;; [vnegq_f])
 270 ;;
 271 (define_insn "mve_vnegq_f<mode>"
 272   [
 273    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 274         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 275          VNEGQ_F))
 276   ]
 277   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 278   "vneg.f%#<V_sz_elem>  %q0, %q1"
 279   [(set_attr "type" "mve_move")
 280 ])
 281
 282 ;;
 283 ;; [vdupq_n_f])
 284 ;;
 285 (define_insn "mve_vdupq_n_f<mode>"
 286   [
 287    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 288         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 289          VDUPQ_N_F))
 290   ]
 291   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 292   "vdup.%#<V_sz_elem>   %q0, %1"
 293   [(set_attr "type" "mve_move")
 294 ])
 295
 296 ;;
 297 ;; [vabsq_f])
 298 ;;
 299 (define_insn "mve_vabsq_f<mode>"
 300   [
 301    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 302         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 303          VABSQ_F))
 304   ]
 305   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 306   "vabs.f%#<V_sz_elem>  %q0, %q1"
 307   [(set_attr "type" "mve_move")
 308 ])
 309
 310 ;;
 311 ;; [vrev32q_f])
 312 ;;
 313 (define_insn "mve_vrev32q_fv8hf"
 314   [
 315    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 316         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 317          VREV32Q_F))
 318   ]
 319   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 320   "vrev32.16 %q0, %q1"
 321   [(set_attr "type" "mve_move")
 322 ])
 323 ;;
 324 ;; [vcvttq_f32_f16])
 325 ;;
 326 (define_insn "mve_vcvttq_f32_f16v4sf"
 327   [
 328    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 329         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 330          VCVTTQ_F32_F16))
 331   ]
 332   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 333   "vcvtt.f32.f16 %q0, %q1"
 334   [(set_attr "type" "mve_move")
 335 ])
 336
 337 ;;
 338 ;; [vcvtbq_f32_f16])
 339 ;;
 340 (define_insn "mve_vcvtbq_f32_f16v4sf"
 341   [
 342    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 343         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 344          VCVTBQ_F32_F16))
 345   ]
 346   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 347   "vcvtb.f32.f16 %q0, %q1"
 348   [(set_attr "type" "mve_move")
 349 ])
 350
 351 ;;
 352 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 353 ;;
 354 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 355   [
 356    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 357         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 358          VCVTQ_TO_F))
 359   ]
 360   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 361   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 362   [(set_attr "type" "mve_move")
 363 ])
 364
 365 ;;
 366 ;; [vrev64q_u, vrev64q_s])
 367 ;;
 368 (define_insn "mve_vrev64q_<supf><mode>"
 369   [
 370    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 371         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 372          VREV64Q))
 373   ]
 374   "TARGET_HAVE_MVE"
 375   "vrev64.%#<V_sz_elem> %q0, %q1"
 376   [(set_attr "type" "mve_move")
 377 ])
 378
 379 ;;
 380 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 381 ;;
 382 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 383   [
 384    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 385         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 386          VCVTQ_FROM_F))
 387   ]
 388   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 389   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 390   [(set_attr "type" "mve_move")
 391 ])
 392 ;; [vqnegq_s])
 393 ;;
 394 (define_insn "mve_vqnegq_s<mode>"
 395   [
 396    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 397         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 398          VQNEGQ_S))
 399   ]
 400   "TARGET_HAVE_MVE"
 401   "vqneg.s%#<V_sz_elem> %q0, %q1"
 402   [(set_attr "type" "mve_move")
 403 ])
 404
 405 ;;
 406 ;; [vqabsq_s])
 407 ;;
 408 (define_insn "mve_vqabsq_s<mode>"
 409   [
 410    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 411         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 412          VQABSQ_S))
 413   ]
 414   "TARGET_HAVE_MVE"
 415   "vqabs.s%#<V_sz_elem> %q0, %q1"
 416   [(set_attr "type" "mve_move")
 417 ])
 418
 419 ;;
 420 ;; [vnegq_s])
 421 ;;
 422 (define_insn "mve_vnegq_s<mode>"
 423   [
 424    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 425         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 426          VNEGQ_S))
 427   ]
 428   "TARGET_HAVE_MVE"
 429   "vneg.s%#<V_sz_elem>  %q0, %q1"
 430   [(set_attr "type" "mve_move")
 431 ])
 432
 433 ;;
 434 ;; [vmvnq_u, vmvnq_s])
 435 ;;
 436 (define_insn "mve_vmvnq_<supf><mode>"
 437   [
 438    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 439         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 440          VMVNQ))
 441   ]
 442   "TARGET_HAVE_MVE"
 443   "vmvn %q0, %q1"
 444   [(set_attr "type" "mve_move")
 445 ])
 446
 447 ;;
 448 ;; [vdupq_n_u, vdupq_n_s])
 449 ;;
 450 (define_insn "mve_vdupq_n_<supf><mode>"
 451   [
 452    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 453         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 454          VDUPQ_N))
 455   ]
 456   "TARGET_HAVE_MVE"
 457   "vdup.%#<V_sz_elem>   %q0, %1"
 458   [(set_attr "type" "mve_move")
 459 ])
 460
 461 ;;
 462 ;; [vclzq_u, vclzq_s])
 463 ;;
 464 (define_insn "mve_vclzq_<supf><mode>"
 465   [
 466    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 467         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 468          VCLZQ))
 469   ]
 470   "TARGET_HAVE_MVE"
 471   "vclz.i%#<V_sz_elem>  %q0, %q1"
 472   [(set_attr "type" "mve_move")
 473 ])
 474
 475 ;;
 476 ;; [vclsq_s])
 477 ;;
 478 (define_insn "mve_vclsq_s<mode>"
 479   [
 480    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 481         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 482          VCLSQ_S))
 483   ]
 484   "TARGET_HAVE_MVE"
 485   "vcls.s%#<V_sz_elem>  %q0, %q1"
 486   [(set_attr "type" "mve_move")
 487 ])
 488
 489 ;;
 490 ;; [vaddvq_s, vaddvq_u])
 491 ;;
 492 (define_insn "mve_vaddvq_<supf><mode>"
 493   [
 494    (set (match_operand:SI 0 "s_register_operand" "=Te")
 495         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 496          VADDVQ))
 497   ]
 498   "TARGET_HAVE_MVE"
 499   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 500   [(set_attr "type" "mve_move")
 501 ])
 502
 503 ;;
 504 ;; [vabsq_s])
 505 ;;
 506 (define_insn "mve_vabsq_s<mode>"
 507   [
 508    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 509         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 510          VABSQ_S))
 511   ]
 512   "TARGET_HAVE_MVE"
 513   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 514   [(set_attr "type" "mve_move")
 515 ])
 516
 517 ;;
 518 ;; [vrev32q_u, vrev32q_s])
 519 ;;
 520 (define_insn "mve_vrev32q_<supf><mode>"
 521   [
 522    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 523         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 524          VREV32Q))
 525   ]
 526   "TARGET_HAVE_MVE"
 527   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 528   [(set_attr "type" "mve_move")
 529 ])
 530
 531 ;;
 532 ;; [vmovltq_u, vmovltq_s])
 533 ;;
 534 (define_insn "mve_vmovltq_<supf><mode>"
 535   [
 536    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 537         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 538          VMOVLTQ))
 539   ]
 540   "TARGET_HAVE_MVE"
 541   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 542   [(set_attr "type" "mve_move")
 543 ])
 544
 545 ;;
 546 ;; [vmovlbq_s, vmovlbq_u])
 547 ;;
 548 (define_insn "mve_vmovlbq_<supf><mode>"
 549   [
 550    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 551         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 552          VMOVLBQ))
 553   ]
 554   "TARGET_HAVE_MVE"
 555   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 556   [(set_attr "type" "mve_move")
 557 ])
 558
 559 ;;
 560 ;; [vcvtpq_s, vcvtpq_u])
 561 ;;
 562 (define_insn "mve_vcvtpq_<supf><mode>"
 563   [
 564    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 565         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 566          VCVTPQ))
 567   ]
 568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 569   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 570   [(set_attr "type" "mve_move")
 571 ])
 572
 573 ;;
 574 ;; [vcvtnq_s, vcvtnq_u])
 575 ;;
 576 (define_insn "mve_vcvtnq_<supf><mode>"
 577   [
 578    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 579         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 580          VCVTNQ))
 581   ]
 582   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 583   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 584   [(set_attr "type" "mve_move")
 585 ])
 586
 587 ;;
 588 ;; [vcvtmq_s, vcvtmq_u])
 589 ;;
 590 (define_insn "mve_vcvtmq_<supf><mode>"
 591   [
 592    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 593         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 594          VCVTMQ))
 595   ]
 596   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 597   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 598   [(set_attr "type" "mve_move")
 599 ])
 600
 601 ;;
 602 ;; [vcvtaq_u, vcvtaq_s])
 603 ;;
 604 (define_insn "mve_vcvtaq_<supf><mode>"
 605   [
 606    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 607         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 608          VCVTAQ))
 609   ]
 610   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 611   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 612   [(set_attr "type" "mve_move")
 613 ])
 614
 615 ;;
 616 ;; [vmvnq_n_u, vmvnq_n_s])
 617 ;;
 618 (define_insn "mve_vmvnq_n_<supf><mode>"
 619   [
 620    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 621         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 622          VMVNQ_N))
 623   ]
 624   "TARGET_HAVE_MVE"
 625   "vmvn.i%#<V_sz_elem>  %q0, %1"
 626   [(set_attr "type" "mve_move")
 627 ])
 628
 629 ;;
 630 ;; [vrev16q_u, vrev16q_s])
 631 ;;
 632 (define_insn "mve_vrev16q_<supf>v16qi"
 633   [
 634    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 635         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 636          VREV16Q))
 637   ]
 638   "TARGET_HAVE_MVE"
 639   "vrev16.8 %q0, %q1"
 640   [(set_attr "type" "mve_move")
 641 ])
 642
 643 ;;
 644 ;; [vaddlvq_s vaddlvq_u])
 645 ;;
 646 (define_insn "mve_vaddlvq_<supf>v4si"
 647   [
 648    (set (match_operand:DI 0 "s_register_operand" "=r")
 649         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 650          VADDLVQ))
 651   ]
 652   "TARGET_HAVE_MVE"
 653   "vaddlv.<supf>32 %Q0, %R0, %q1"
 654   [(set_attr "type" "mve_move")
 655 ])
 656
 657 ;;
 658 ;; [vctp8q vctp16q vctp32q vctp64q])
 659 ;;
 660 (define_insn "mve_vctp<mode1>qhi"
 661   [
 662    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 663         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 664         VCTPQ))
 665   ]
 666   "TARGET_HAVE_MVE"
 667   "vctp.<mode1> %1"
 668   [(set_attr "type" "mve_move")
 669 ])
 670
 671 ;;
 672 ;; [vpnot])
 673 ;;
 674 (define_insn "mve_vpnothi"
 675   [
 676    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 677         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 678          VPNOT))
 679   ]
 680   "TARGET_HAVE_MVE"
 681   "vpnot"
 682   [(set_attr "type" "mve_move")
 683 ])
 684
 685 ;;
 686 ;; [vsubq_n_f])
 687 ;;
 688 (define_insn "mve_vsubq_n_f<mode>"
 689   [
 690    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 691         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 692                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 693          VSUBQ_N_F))
 694   ]
 695   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 696   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 697   [(set_attr "type" "mve_move")
 698 ])
 699
 700 ;;
 701 ;; [vbrsrq_n_f])
 702 ;;
 703 (define_insn "mve_vbrsrq_n_f<mode>"
 704   [
 705    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 706         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 707                        (match_operand:SI 2 "s_register_operand" "r")]
 708          VBRSRQ_N_F))
 709   ]
 710   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 711   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 712   [(set_attr "type" "mve_move")
 713 ])
 714
 715 ;;
 716 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 717 ;;
 718 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 719   [
 720    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 721         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 722                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 723          VCVTQ_N_TO_F))
 724   ]
 725   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 726   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 727   [(set_attr "type" "mve_move")
 728 ])
 729
 730 ;; [vcreateq_f])
 731 ;;
 732 (define_insn "mve_vcreateq_f<mode>"
 733   [
 734    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 735         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 736                        (match_operand:DI 2 "s_register_operand" "r")]
 737          VCREATEQ_F))
 738   ]
 739   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 740   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 741   [(set_attr "type" "mve_move")
 742    (set_attr "length""8")])
 743
 744 ;;
 745 ;; [vcreateq_u, vcreateq_s])
 746 ;;
 747 (define_insn "mve_vcreateq_<supf><mode>"
 748   [
 749    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 750         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 751                        (match_operand:DI 2 "s_register_operand" "r")]
 752          VCREATEQ))
 753   ]
 754   "TARGET_HAVE_MVE"
 755   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 756   [(set_attr "type" "mve_move")
 757    (set_attr "length""8")])
 758
 759 ;;
 760 ;; [vshrq_n_s, vshrq_n_u])
 761 ;;
 762 (define_insn "mve_vshrq_n_<supf><mode>"
 763   [
 764    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 765         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 766                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 767          VSHRQ_N))
 768   ]
 769   "TARGET_HAVE_MVE"
 770   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 771   [(set_attr "type" "mve_move")
 772 ])
 773
 774 ;;
 775 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 776 ;;
 777 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 778   [
 779    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 780         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 781                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 782          VCVTQ_N_FROM_F))
 783   ]
 784   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 785   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 786   [(set_attr "type" "mve_move")
 787 ])
 788
 789 ;;
 790 ;; [vaddlvq_p_s])
 791 ;;
 792 (define_insn "mve_vaddlvq_p_<supf>v4si"
 793   [
 794    (set (match_operand:DI 0 "s_register_operand" "=r")
 795         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 796                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 797          VADDLVQ_P))
 798   ]
 799   "TARGET_HAVE_MVE"
 800   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 801   [(set_attr "type" "mve_move")
 802    (set_attr "length""8")])
 803
 804 ;;
 805 ;; [vcmpneq_u, vcmpneq_s])
 806 ;;
 807 (define_insn "mve_vcmpneq_<supf><mode>"
 808   [
 809    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 810         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 811                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 812          VCMPNEQ))
 813   ]
 814   "TARGET_HAVE_MVE"
 815   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
 816   [(set_attr "type" "mve_move")
 817 ])
 818
 819 ;;
 820 ;; [vshlq_s, vshlq_u])
 821 ;;
 822 (define_insn "mve_vshlq_<supf><mode>"
 823   [
 824    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 825         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 826                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 827          VSHLQ))
 828   ]
 829   "TARGET_HAVE_MVE"
 830   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
 831   [(set_attr "type" "mve_move")
 832 ])
 833
 834 ;;
 835 ;; [vabdq_s, vabdq_u])
 836 ;;
 837 (define_insn "mve_vabdq_<supf><mode>"
 838   [
 839    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 840         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 841                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 842          VABDQ))
 843   ]
 844   "TARGET_HAVE_MVE"
 845   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 846   [(set_attr "type" "mve_move")
 847 ])
 848
 849 ;;
 850 ;; [vaddq_n_s, vaddq_n_u])
 851 ;;
 852 (define_insn "mve_vaddq_n_<supf><mode>"
 853   [
 854    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 855         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 856                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 857          VADDQ_N))
 858   ]
 859   "TARGET_HAVE_MVE"
 860   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 861   [(set_attr "type" "mve_move")
 862 ])
 863
 864 ;;
 865 ;; [vaddvaq_s, vaddvaq_u])
 866 ;;
 867 (define_insn "mve_vaddvaq_<supf><mode>"
 868   [
 869    (set (match_operand:SI 0 "s_register_operand" "=Te")
 870         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 871                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 872          VADDVAQ))
 873   ]
 874   "TARGET_HAVE_MVE"
 875   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 876   [(set_attr "type" "mve_move")
 877 ])
 878
 879 ;;
 880 ;; [vaddvq_p_u, vaddvq_p_s])
 881 ;;
 882 (define_insn "mve_vaddvq_p_<supf><mode>"
 883   [
 884    (set (match_operand:SI 0 "s_register_operand" "=Te")
 885         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 886                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 887          VADDVQ_P))
 888   ]
 889   "TARGET_HAVE_MVE"
 890   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 891   [(set_attr "type" "mve_move")
 892    (set_attr "length""8")])
 893
 894 ;;
 895 ;; [vandq_u, vandq_s])
 896 ;;
 897 ;; signed and unsigned versions are the same: define the unsigned
 898 ;; insn, and use an expander for the signed one as we still reference
 899 ;; both names from arm_mve.h.
 900 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 901 (define_insn "mve_vandq_u<mode>"
 902   [
 903    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 904         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 905                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 906   ]
 907   "TARGET_HAVE_MVE"
 908   "@
 909    vand\t%q0, %q1, %q2
 910    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 911   [(set_attr "type" "mve_move")
 912 ])
 913 (define_expand "mve_vandq_s<mode>"
 914   [
 915    (set (match_operand:MVE_2 0 "s_register_operand")
 916         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 917                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 918   ]
 919   "TARGET_HAVE_MVE"
 920 )
 921
 922 ;;
 923 ;; [vbicq_s, vbicq_u])
 924 ;;
 925 (define_insn "mve_vbicq_<supf><mode>"
 926   [
 927    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 928         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 929                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 930          VBICQ))
 931   ]
 932   "TARGET_HAVE_MVE"
 933   "vbic %q0, %q1, %q2"
 934   [(set_attr "type" "mve_move")
 935 ])
 936
 937 ;;
 938 ;; [vbrsrq_n_u, vbrsrq_n_s])
 939 ;;
 940 (define_insn "mve_vbrsrq_n_<supf><mode>"
 941   [
 942    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 943         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 944                        (match_operand:SI 2 "s_register_operand" "r")]
 945          VBRSRQ_N))
 946   ]
 947   "TARGET_HAVE_MVE"
 948   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 949   [(set_attr "type" "mve_move")
 950 ])
 951
 952 ;;
 953 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
 954 ;;
 955 (define_insn "mve_vcaddq_rot270_<supf><mode>"
 956   [
 957    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 958         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 959                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 960          VCADDQ_ROT270))
 961   ]
 962   "TARGET_HAVE_MVE"
 963   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
 964   [(set_attr "type" "mve_move")
 965 ])
 966
 967 ;;
 968 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
 969 ;;
 970 (define_insn "mve_vcaddq_rot90_<supf><mode>"
 971   [
 972    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 973         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 974                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 975          VCADDQ_ROT90))
 976   ]
 977   "TARGET_HAVE_MVE"
 978   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
 979   [(set_attr "type" "mve_move")
 980 ])
 981
 982 ;;
 983 ;; [vcmpcsq_n_u])
 984 ;;
 985 (define_insn "mve_vcmpcsq_n_u<mode>"
 986   [
 987    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 988         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
 989                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
 990          VCMPCSQ_N_U))
 991   ]
 992   "TARGET_HAVE_MVE"
 993   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
 994   [(set_attr "type" "mve_move")
 995 ])
 996
 997 ;;
 998 ;; [vcmpcsq_u])
 999 ;;
1000 (define_insn "mve_vcmpcsq_u<mode>"
1001   [
1002    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1003         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1004                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1005          VCMPCSQ_U))
1006   ]
1007   "TARGET_HAVE_MVE"
1008   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1009   [(set_attr "type" "mve_move")
1010 ])
1011
1012 ;;
1013 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1014 ;;
1015 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1016   [
1017    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1018         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1019                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1020          VCMPEQQ_N))
1021   ]
1022   "TARGET_HAVE_MVE"
1023   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1024   [(set_attr "type" "mve_move")
1025 ])
1026
1027 ;;
1028 ;; [vcmpeqq_u, vcmpeqq_s])
1029 ;;
1030 (define_insn "mve_vcmpeqq_<supf><mode>"
1031   [
1032    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1033         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1034                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1035          VCMPEQQ))
1036   ]
1037   "TARGET_HAVE_MVE"
1038   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1039   [(set_attr "type" "mve_move")
1040 ])
1041
1042 ;;
1043 ;; [vcmpgeq_n_s])
1044 ;;
1045 (define_insn "mve_vcmpgeq_n_s<mode>"
1046   [
1047    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1048         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1049                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1050          VCMPGEQ_N_S))
1051   ]
1052   "TARGET_HAVE_MVE"
1053   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1054   [(set_attr "type" "mve_move")
1055 ])
1056
1057 ;;
1058 ;; [vcmpgeq_s])
1059 ;;
1060 (define_insn "mve_vcmpgeq_s<mode>"
1061   [
1062    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1063         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1064                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1065          VCMPGEQ_S))
1066   ]
1067   "TARGET_HAVE_MVE"
1068   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1069   [(set_attr "type" "mve_move")
1070 ])
1071
1072 ;;
1073 ;; [vcmpgtq_n_s])
1074 ;;
1075 (define_insn "mve_vcmpgtq_n_s<mode>"
1076   [
1077    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1078         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1079                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1080          VCMPGTQ_N_S))
1081   ]
1082   "TARGET_HAVE_MVE"
1083   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1084   [(set_attr "type" "mve_move")
1085 ])
1086
1087 ;;
1088 ;; [vcmpgtq_s])
1089 ;;
1090 (define_insn "mve_vcmpgtq_s<mode>"
1091   [
1092    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1093         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1094                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1095          VCMPGTQ_S))
1096   ]
1097   "TARGET_HAVE_MVE"
1098   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1099   [(set_attr "type" "mve_move")
1100 ])
1101
1102 ;;
1103 ;; [vcmphiq_n_u])
1104 ;;
1105 (define_insn "mve_vcmphiq_n_u<mode>"
1106   [
1107    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1108         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1109                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1110          VCMPHIQ_N_U))
1111   ]
1112   "TARGET_HAVE_MVE"
1113   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1114   [(set_attr "type" "mve_move")
1115 ])
1116
1117 ;;
1118 ;; [vcmphiq_u])
1119 ;;
1120 (define_insn "mve_vcmphiq_u<mode>"
1121   [
1122    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1123         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1124                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1125          VCMPHIQ_U))
1126   ]
1127   "TARGET_HAVE_MVE"
1128   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1129   [(set_attr "type" "mve_move")
1130 ])
1131
1132 ;;
1133 ;; [vcmpleq_n_s])
1134 ;;
1135 (define_insn "mve_vcmpleq_n_s<mode>"
1136   [
1137    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1138         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1139                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1140          VCMPLEQ_N_S))
1141   ]
1142   "TARGET_HAVE_MVE"
1143   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1144   [(set_attr "type" "mve_move")
1145 ])
1146
1147 ;;
1148 ;; [vcmpleq_s])
1149 ;;
1150 (define_insn "mve_vcmpleq_s<mode>"
1151   [
1152    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1153         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1154                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1155          VCMPLEQ_S))
1156   ]
1157   "TARGET_HAVE_MVE"
1158   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1159   [(set_attr "type" "mve_move")
1160 ])
1161
1162 ;;
1163 ;; [vcmpltq_n_s])
1164 ;;
1165 (define_insn "mve_vcmpltq_n_s<mode>"
1166   [
1167    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1168         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1169                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1170          VCMPLTQ_N_S))
1171   ]
1172   "TARGET_HAVE_MVE"
1173   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1174   [(set_attr "type" "mve_move")
1175 ])
1176
1177 ;;
1178 ;; [vcmpltq_s])
1179 ;;
1180 (define_insn "mve_vcmpltq_s<mode>"
1181   [
1182    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1183         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1184                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1185          VCMPLTQ_S))
1186   ]
1187   "TARGET_HAVE_MVE"
1188   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1189   [(set_attr "type" "mve_move")
1190 ])
1191
1192 ;;
1193 ;; [vcmpneq_n_u, vcmpneq_n_s])
1194 ;;
1195 (define_insn "mve_vcmpneq_n_<supf><mode>"
1196   [
1197    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1198         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1199                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1200          VCMPNEQ_N))
1201   ]
1202   "TARGET_HAVE_MVE"
1203   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1204   [(set_attr "type" "mve_move")
1205 ])
1206
1207 ;;
1208 ;; [veorq_u, veorq_s])
1209 ;;
1210 (define_insn "mve_veorq_<supf><mode>"
1211   [
1212    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1213         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1214                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1215          VEORQ))
1216   ]
1217   "TARGET_HAVE_MVE"
1218   "veor %q0, %q1, %q2"
1219   [(set_attr "type" "mve_move")
1220 ])
1221
1222 ;;
1223 ;; [vhaddq_n_u, vhaddq_n_s])
1224 ;;
1225 (define_insn "mve_vhaddq_n_<supf><mode>"
1226   [
1227    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1228         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1229                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1230          VHADDQ_N))
1231   ]
1232   "TARGET_HAVE_MVE"
1233   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1234   [(set_attr "type" "mve_move")
1235 ])
1236
1237 ;;
1238 ;; [vhaddq_s, vhaddq_u])
1239 ;;
1240 (define_insn "mve_vhaddq_<supf><mode>"
1241   [
1242    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1243         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1244                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1245          VHADDQ))
1246   ]
1247   "TARGET_HAVE_MVE"
1248   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1249   [(set_attr "type" "mve_move")
1250 ])
1251
1252 ;;
1253 ;; [vhcaddq_rot270_s])
1254 ;;
1255 (define_insn "mve_vhcaddq_rot270_s<mode>"
1256   [
1257    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1258         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1259                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1260          VHCADDQ_ROT270_S))
1261   ]
1262   "TARGET_HAVE_MVE"
1263   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1264   [(set_attr "type" "mve_move")
1265 ])
1266
1267 ;;
1268 ;; [vhcaddq_rot90_s])
1269 ;;
1270 (define_insn "mve_vhcaddq_rot90_s<mode>"
1271   [
1272    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1273         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1274                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1275          VHCADDQ_ROT90_S))
1276   ]
1277   "TARGET_HAVE_MVE"
1278   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1279   [(set_attr "type" "mve_move")
1280 ])
1281
1282 ;;
1283 ;; [vhsubq_n_u, vhsubq_n_s])
1284 ;;
1285 (define_insn "mve_vhsubq_n_<supf><mode>"
1286   [
1287    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1288         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1289                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1290          VHSUBQ_N))
1291   ]
1292   "TARGET_HAVE_MVE"
1293   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1294   [(set_attr "type" "mve_move")
1295 ])
1296
1297 ;;
1298 ;; [vhsubq_s, vhsubq_u])
1299 ;;
1300 (define_insn "mve_vhsubq_<supf><mode>"
1301   [
1302    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1303         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1304                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1305          VHSUBQ))
1306   ]
1307   "TARGET_HAVE_MVE"
1308   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1309   [(set_attr "type" "mve_move")
1310 ])
1311
1312 ;;
1313 ;; [vmaxaq_s])
1314 ;;
1315 (define_insn "mve_vmaxaq_s<mode>"
1316   [
1317    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1318         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1319                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1320          VMAXAQ_S))
1321   ]
1322   "TARGET_HAVE_MVE"
1323   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1324   [(set_attr "type" "mve_move")
1325 ])
1326
1327 ;;
1328 ;; [vmaxavq_s])
1329 ;;
1330 (define_insn "mve_vmaxavq_s<mode>"
1331   [
1332    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1333         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1334                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1335          VMAXAVQ_S))
1336   ]
1337   "TARGET_HAVE_MVE"
1338   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1339   [(set_attr "type" "mve_move")
1340 ])
1341
1342 ;;
1343 ;; [vmaxq_u, vmaxq_s])
1344 ;;
1345 (define_insn "mve_vmaxq_s<mode>"
1346   [
1347    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1348         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1349                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1350   ]
1351   "TARGET_HAVE_MVE"
1352   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1353   [(set_attr "type" "mve_move")
1354 ])
1355
1356 (define_insn "mve_vmaxq_u<mode>"
1357   [
1358    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1359         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1360                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1361   ]
1362   "TARGET_HAVE_MVE"
1363   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1364   [(set_attr "type" "mve_move")
1365 ])
1366
1367 ;;
1368 ;; [vmaxvq_u, vmaxvq_s])
1369 ;;
1370 (define_insn "mve_vmaxvq_<supf><mode>"
1371   [
1372    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1373         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1374                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1375          VMAXVQ))
1376   ]
1377   "TARGET_HAVE_MVE"
1378   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1379   [(set_attr "type" "mve_move")
1380 ])
1381
1382 ;;
1383 ;; [vminaq_s])
1384 ;;
1385 (define_insn "mve_vminaq_s<mode>"
1386   [
1387    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1388         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1389                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1390          VMINAQ_S))
1391   ]
1392   "TARGET_HAVE_MVE"
1393   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1394   [(set_attr "type" "mve_move")
1395 ])
1396
1397 ;;
1398 ;; [vminavq_s])
1399 ;;
1400 (define_insn "mve_vminavq_s<mode>"
1401   [
1402    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1403         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1404                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1405          VMINAVQ_S))
1406   ]
1407   "TARGET_HAVE_MVE"
1408   "vminav.s%#<V_sz_elem>\t%0, %q2"
1409   [(set_attr "type" "mve_move")
1410 ])
1411
1412 ;;
1413 ;; [vminq_s, vminq_u])
1414 ;;
1415 (define_insn "mve_vminq_s<mode>"
1416   [
1417    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1418         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1419                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1420   ]
1421   "TARGET_HAVE_MVE"
1422   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1423   [(set_attr "type" "mve_move")
1424 ])
1425
1426 (define_insn "mve_vminq_u<mode>"
1427   [
1428    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1429         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1430                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1431   ]
1432   "TARGET_HAVE_MVE"
1433   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1434   [(set_attr "type" "mve_move")
1435 ])
1436
1437 ;;
1438 ;; [vminvq_u, vminvq_s])
1439 ;;
1440 (define_insn "mve_vminvq_<supf><mode>"
1441   [
1442    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1443         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1444                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1445          VMINVQ))
1446   ]
1447   "TARGET_HAVE_MVE"
1448   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1449   [(set_attr "type" "mve_move")
1450 ])
1451
1452 ;;
1453 ;; [vmladavq_u, vmladavq_s])
1454 ;;
1455 (define_insn "mve_vmladavq_<supf><mode>"
1456   [
1457    (set (match_operand:SI 0 "s_register_operand" "=Te")
1458         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1459                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1460          VMLADAVQ))
1461   ]
1462   "TARGET_HAVE_MVE"
1463   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1464   [(set_attr "type" "mve_move")
1465 ])
1466
1467 ;;
1468 ;; [vmladavxq_s])
1469 ;;
1470 (define_insn "mve_vmladavxq_s<mode>"
1471   [
1472    (set (match_operand:SI 0 "s_register_operand" "=Te")
1473         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1474                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1475          VMLADAVXQ_S))
1476   ]
1477   "TARGET_HAVE_MVE"
1478   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1479   [(set_attr "type" "mve_move")
1480 ])
1481
1482 ;;
1483 ;; [vmlsdavq_s])
1484 ;;
1485 (define_insn "mve_vmlsdavq_s<mode>"
1486   [
1487    (set (match_operand:SI 0 "s_register_operand" "=Te")
1488         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1489                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1490          VMLSDAVQ_S))
1491   ]
1492   "TARGET_HAVE_MVE"
1493   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1494   [(set_attr "type" "mve_move")
1495 ])
1496
1497 ;;
1498 ;; [vmlsdavxq_s])
1499 ;;
1500 (define_insn "mve_vmlsdavxq_s<mode>"
1501   [
1502    (set (match_operand:SI 0 "s_register_operand" "=Te")
1503         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1504                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1505          VMLSDAVXQ_S))
1506   ]
1507   "TARGET_HAVE_MVE"
1508   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1509   [(set_attr "type" "mve_move")
1510 ])
1511
1512 ;;
1513 ;; [vmulhq_s, vmulhq_u])
1514 ;;
1515 (define_insn "mve_vmulhq_<supf><mode>"
1516   [
1517    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1518         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1519                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1520          VMULHQ))
1521   ]
1522   "TARGET_HAVE_MVE"
1523   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1524   [(set_attr "type" "mve_move")
1525 ])
1526
1527 ;;
1528 ;; [vmullbq_int_u, vmullbq_int_s])
1529 ;;
1530 (define_insn "mve_vmullbq_int_<supf><mode>"
1531   [
1532    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1533         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1534                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1535          VMULLBQ_INT))
1536   ]
1537   "TARGET_HAVE_MVE"
1538   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1539   [(set_attr "type" "mve_move")
1540 ])
1541
1542 ;;
1543 ;; [vmulltq_int_u, vmulltq_int_s])
1544 ;;
1545 (define_insn "mve_vmulltq_int_<supf><mode>"
1546   [
1547    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1548         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1549                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1550          VMULLTQ_INT))
1551   ]
1552   "TARGET_HAVE_MVE"
1553   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1554   [(set_attr "type" "mve_move")
1555 ])
1556
1557 ;;
1558 ;; [vmulq_n_u, vmulq_n_s])
1559 ;;
1560 (define_insn "mve_vmulq_n_<supf><mode>"
1561   [
1562    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1563         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1564                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1565          VMULQ_N))
1566   ]
1567   "TARGET_HAVE_MVE"
1568   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1569   [(set_attr "type" "mve_move")
1570 ])
1571
1572 ;;
1573 ;; [vmulq_u, vmulq_s])
1574 ;;
1575 (define_insn "mve_vmulq_<supf><mode>"
1576   [
1577    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1578         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1579                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1580          VMULQ))
1581   ]
1582   "TARGET_HAVE_MVE"
1583   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1584   [(set_attr "type" "mve_move")
1585 ])
1586
1587 (define_insn "mve_vmulq<mode>"
1588   [
1589    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1590         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1591                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1592   ]
1593   "TARGET_HAVE_MVE"
1594   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1595   [(set_attr "type" "mve_move")
1596 ])
1597
1598 ;;
1599 ;; [vornq_u, vornq_s])
1600 ;;
1601 (define_insn "mve_vornq_<supf><mode>"
1602   [
1603    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1604         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1605                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1606          VORNQ))
1607   ]
1608   "TARGET_HAVE_MVE"
1609   "vorn %q0, %q1, %q2"
1610   [(set_attr "type" "mve_move")
1611 ])
1612
1613 ;;
1614 ;; [vorrq_s, vorrq_u])
1615 ;;
1616 ;; signed and unsigned versions are the same: define the unsigned
1617 ;; insn, and use an expander for the signed one as we still reference
1618 ;; both names from arm_mve.h.
1619 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1620 (define_insn "mve_vorrq_s<mode>"
1621   [
1622    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1623         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1624                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1625   ]
1626   "TARGET_HAVE_MVE"
1627   "@
1628    vorr\t%q0, %q1, %q2
1629    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1630   [(set_attr "type" "mve_move")
1631 ])
1632 (define_expand "mve_vorrq_u<mode>"
1633   [
1634    (set (match_operand:MVE_2 0 "s_register_operand")
1635         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1636                    (match_operand:MVE_2 2 "neon_logic_op2")))
1637   ]
1638   "TARGET_HAVE_MVE"
1639 )
1640
1641 ;;
1642 ;; [vqaddq_n_s, vqaddq_n_u])
1643 ;;
1644 (define_insn "mve_vqaddq_n_<supf><mode>"
1645   [
1646    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1647         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1648                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1649          VQADDQ_N))
1650   ]
1651   "TARGET_HAVE_MVE"
1652   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1653   [(set_attr "type" "mve_move")
1654 ])
1655
1656 ;;
1657 ;; [vqaddq_u, vqaddq_s])
1658 ;;
1659 (define_insn "mve_vqaddq_<supf><mode>"
1660   [
1661    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1662         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1663                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1664          VQADDQ))
1665   ]
1666   "TARGET_HAVE_MVE"
1667   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1668   [(set_attr "type" "mve_move")
1669 ])
1670
1671 ;;
1672 ;; [vqdmulhq_n_s])
1673 ;;
1674 (define_insn "mve_vqdmulhq_n_s<mode>"
1675   [
1676    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1677         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1678                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1679          VQDMULHQ_N_S))
1680   ]
1681   "TARGET_HAVE_MVE"
1682   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1683   [(set_attr "type" "mve_move")
1684 ])
1685
1686 ;;
1687 ;; [vqdmulhq_s])
1688 ;;
1689 (define_insn "mve_vqdmulhq_s<mode>"
1690   [
1691    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1692         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1693                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1694          VQDMULHQ_S))
1695   ]
1696   "TARGET_HAVE_MVE"
1697   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1698   [(set_attr "type" "mve_move")
1699 ])
1700
1701 ;;
1702 ;; [vqrdmulhq_n_s])
1703 ;;
1704 (define_insn "mve_vqrdmulhq_n_s<mode>"
1705   [
1706    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1707         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1708                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1709          VQRDMULHQ_N_S))
1710   ]
1711   "TARGET_HAVE_MVE"
1712   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1713   [(set_attr "type" "mve_move")
1714 ])
1715
1716 ;;
1717 ;; [vqrdmulhq_s])
1718 ;;
1719 (define_insn "mve_vqrdmulhq_s<mode>"
1720   [
1721    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1722         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1723                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1724          VQRDMULHQ_S))
1725   ]
1726   "TARGET_HAVE_MVE"
1727   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1728   [(set_attr "type" "mve_move")
1729 ])
1730
1731 ;;
1732 ;; [vqrshlq_n_s, vqrshlq_n_u])
1733 ;;
1734 (define_insn "mve_vqrshlq_n_<supf><mode>"
1735   [
1736    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1737         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1738                        (match_operand:SI 2 "s_register_operand" "r")]
1739          VQRSHLQ_N))
1740   ]
1741   "TARGET_HAVE_MVE"
1742   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1743   [(set_attr "type" "mve_move")
1744 ])
1745
1746 ;;
1747 ;; [vqrshlq_s, vqrshlq_u])
1748 ;;
1749 (define_insn "mve_vqrshlq_<supf><mode>"
1750   [
1751    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1752         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1753                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1754          VQRSHLQ))
1755   ]
1756   "TARGET_HAVE_MVE"
1757   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1758   [(set_attr "type" "mve_move")
1759 ])
1760
1761 ;;
1762 ;; [vqshlq_n_s, vqshlq_n_u])
1763 ;;
1764 (define_insn "mve_vqshlq_n_<supf><mode>"
1765   [
1766    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1767         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1768                        (match_operand:SI 2 "immediate_operand" "i")]
1769          VQSHLQ_N))
1770   ]
1771   "TARGET_HAVE_MVE"
1772   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1773   [(set_attr "type" "mve_move")
1774 ])
1775
1776 ;;
1777 ;; [vqshlq_r_u, vqshlq_r_s])
1778 ;;
1779 (define_insn "mve_vqshlq_r_<supf><mode>"
1780   [
1781    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1782         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1783                        (match_operand:SI 2 "s_register_operand" "r")]
1784          VQSHLQ_R))
1785   ]
1786   "TARGET_HAVE_MVE"
1787   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1788   [(set_attr "type" "mve_move")
1789 ])
1790
1791 ;;
1792 ;; [vqshlq_s, vqshlq_u])
1793 ;;
1794 (define_insn "mve_vqshlq_<supf><mode>"
1795   [
1796    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1797         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1798                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1799          VQSHLQ))
1800   ]
1801   "TARGET_HAVE_MVE"
1802   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1803   [(set_attr "type" "mve_move")
1804 ])
1805
1806 ;;
1807 ;; [vqshluq_n_s])
1808 ;;
1809 (define_insn "mve_vqshluq_n_s<mode>"
1810   [
1811    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1812         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1813                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1814          VQSHLUQ_N_S))
1815   ]
1816   "TARGET_HAVE_MVE"
1817   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1818   [(set_attr "type" "mve_move")
1819 ])
1820
1821 ;;
1822 ;; [vqsubq_n_s, vqsubq_n_u])
1823 ;;
1824 (define_insn "mve_vqsubq_n_<supf><mode>"
1825   [
1826    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1827         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1828                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1829          VQSUBQ_N))
1830   ]
1831   "TARGET_HAVE_MVE"
1832   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1833   [(set_attr "type" "mve_move")
1834 ])
1835
1836 ;;
1837 ;; [vqsubq_u, vqsubq_s])
1838 ;;
1839 (define_insn "mve_vqsubq_<supf><mode>"
1840   [
1841    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1842         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1843                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1844          VQSUBQ))
1845   ]
1846   "TARGET_HAVE_MVE"
1847   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1848   [(set_attr "type" "mve_move")
1849 ])
1850
1851 ;;
1852 ;; [vrhaddq_s, vrhaddq_u])
1853 ;;
1854 (define_insn "mve_vrhaddq_<supf><mode>"
1855   [
1856    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1857         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1858                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1859          VRHADDQ))
1860   ]
1861   "TARGET_HAVE_MVE"
1862   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1863   [(set_attr "type" "mve_move")
1864 ])
1865
1866 ;;
1867 ;; [vrmulhq_s, vrmulhq_u])
1868 ;;
1869 (define_insn "mve_vrmulhq_<supf><mode>"
1870   [
1871    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1872         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1873                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1874          VRMULHQ))
1875   ]
1876   "TARGET_HAVE_MVE"
1877   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1878   [(set_attr "type" "mve_move")
1879 ])
1880
1881 ;;
1882 ;; [vrshlq_n_u, vrshlq_n_s])
1883 ;;
1884 (define_insn "mve_vrshlq_n_<supf><mode>"
1885   [
1886    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1887         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1888                        (match_operand:SI 2 "s_register_operand" "r")]
1889          VRSHLQ_N))
1890   ]
1891   "TARGET_HAVE_MVE"
1892   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1893   [(set_attr "type" "mve_move")
1894 ])
1895
1896 ;;
1897 ;; [vrshlq_s, vrshlq_u])
1898 ;;
1899 (define_insn "mve_vrshlq_<supf><mode>"
1900   [
1901    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1902         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1903                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1904          VRSHLQ))
1905   ]
1906   "TARGET_HAVE_MVE"
1907   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1908   [(set_attr "type" "mve_move")
1909 ])
1910
1911 ;;
1912 ;; [vrshrq_n_s, vrshrq_n_u])
1913 ;;
1914 (define_insn "mve_vrshrq_n_<supf><mode>"
1915   [
1916    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1917         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1918                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1919          VRSHRQ_N))
1920   ]
1921   "TARGET_HAVE_MVE"
1922   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1923   [(set_attr "type" "mve_move")
1924 ])
1925
1926 ;;
1927 ;; [vshlq_n_u, vshlq_n_s])
1928 ;;
1929 (define_insn "mve_vshlq_n_<supf><mode>"
1930   [
1931    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1932         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1933                        (match_operand:SI 2 "immediate_operand" "i")]
1934          VSHLQ_N))
1935   ]
1936   "TARGET_HAVE_MVE"
1937   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1938   [(set_attr "type" "mve_move")
1939 ])
1940
1941 ;;
1942 ;; [vshlq_r_s, vshlq_r_u])
1943 ;;
1944 (define_insn "mve_vshlq_r_<supf><mode>"
1945   [
1946    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1947         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1948                        (match_operand:SI 2 "s_register_operand" "r")]
1949          VSHLQ_R))
1950   ]
1951   "TARGET_HAVE_MVE"
1952   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1953   [(set_attr "type" "mve_move")
1954 ])
1955
1956 ;;
1957 ;; [vsubq_n_s, vsubq_n_u])
1958 ;;
1959 (define_insn "mve_vsubq_n_<supf><mode>"
1960   [
1961    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1962         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1963                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1964          VSUBQ_N))
1965   ]
1966   "TARGET_HAVE_MVE"
1967   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1968   [(set_attr "type" "mve_move")
1969 ])
1970
1971 ;;
1972 ;; [vsubq_s, vsubq_u])
1973 ;;
1974 (define_insn "mve_vsubq_<supf><mode>"
1975   [
1976    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1977         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1978                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1979          VSUBQ))
1980   ]
1981   "TARGET_HAVE_MVE"
1982   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1983   [(set_attr "type" "mve_move")
1984 ])
1985
1986 (define_insn "mve_vsubq<mode>"
1987   [
1988    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1989         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1990                      (match_operand:MVE_2 2 "s_register_operand" "w")))
1991   ]
1992   "TARGET_HAVE_MVE"
1993   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1994   [(set_attr "type" "mve_move")
1995 ])
1996
1997 ;;
1998 ;; [vabdq_f])
1999 ;;
2000 (define_insn "mve_vabdq_f<mode>"
2001   [
2002    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2003         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2004                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2005          VABDQ_F))
2006   ]
2007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2008   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2009   [(set_attr "type" "mve_move")
2010 ])
2011
2012 ;;
2013 ;; [vaddlvaq_s vaddlvaq_u])
2014 ;;
2015 (define_insn "mve_vaddlvaq_<supf>v4si"
2016   [
2017    (set (match_operand:DI 0 "s_register_operand" "=r")
2018         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2019                     (match_operand:V4SI 2 "s_register_operand" "w")]
2020          VADDLVAQ))
2021   ]
2022   "TARGET_HAVE_MVE"
2023   "vaddlva.<supf>32 %Q0, %R0, %q2"
2024   [(set_attr "type" "mve_move")
2025 ])
2026
2027 ;;
2028 ;; [vaddq_n_f])
2029 ;;
2030 (define_insn "mve_vaddq_n_f<mode>"
2031   [
2032    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2033         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2034                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2035          VADDQ_N_F))
2036   ]
2037   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2038   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2039   [(set_attr "type" "mve_move")
2040 ])
2041
2042 ;;
2043 ;; [vandq_f])
2044 ;;
2045 (define_insn "mve_vandq_f<mode>"
2046   [
2047    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2048         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2049                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2050   ]
2051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2052   "vand %q0, %q1, %q2"
2053   [(set_attr "type" "mve_move")
2054 ])
2055
2056 ;;
2057 ;; [vbicq_f])
2058 ;;
2059 (define_insn "mve_vbicq_f<mode>"
2060   [
2061    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2062         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2063                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2064          VBICQ_F))
2065   ]
2066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2067   "vbic %q0, %q1, %q2"
2068   [(set_attr "type" "mve_move")
2069 ])
2070
2071 ;;
2072 ;; [vbicq_n_s, vbicq_n_u])
2073 ;;
2074 (define_insn "mve_vbicq_n_<supf><mode>"
2075   [
2076    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2077         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2078                        (match_operand:SI 2 "immediate_operand" "i")]
2079          VBICQ_N))
2080   ]
2081   "TARGET_HAVE_MVE"
2082   "vbic.i%#<V_sz_elem>  %q0, %2"
2083   [(set_attr "type" "mve_move")
2084 ])
2085
2086 ;;
2087 ;; [vcaddq_rot270_f])
2088 ;;
2089 (define_insn "mve_vcaddq_rot270_f<mode>"
2090   [
2091    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2092         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2093                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2094          VCADDQ_ROT270_F))
2095   ]
2096   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2097   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2098   [(set_attr "type" "mve_move")
2099 ])
2100
2101 ;;
2102 ;; [vcaddq_rot90_f])
2103 ;;
2104 (define_insn "mve_vcaddq_rot90_f<mode>"
2105   [
2106    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2107         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2108                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2109          VCADDQ_ROT90_F))
2110   ]
2111   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2112   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2113   [(set_attr "type" "mve_move")
2114 ])
2115
2116 ;;
2117 ;; [vcmpeqq_f])
2118 ;;
2119 (define_insn "mve_vcmpeqq_f<mode>"
2120   [
2121    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2122         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2123                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2124          VCMPEQQ_F))
2125   ]
2126   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2127   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2128   [(set_attr "type" "mve_move")
2129 ])
2130
2131 ;;
2132 ;; [vcmpeqq_n_f])
2133 ;;
2134 (define_insn "mve_vcmpeqq_n_f<mode>"
2135   [
2136    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2137         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2138                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2139          VCMPEQQ_N_F))
2140   ]
2141   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2142   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2143   [(set_attr "type" "mve_move")
2144 ])
2145
2146 ;;
2147 ;; [vcmpgeq_f])
2148 ;;
2149 (define_insn "mve_vcmpgeq_f<mode>"
2150   [
2151    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2152         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2153                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2154          VCMPGEQ_F))
2155   ]
2156   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2157   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2158   [(set_attr "type" "mve_move")
2159 ])
2160
2161 ;;
2162 ;; [vcmpgeq_n_f])
2163 ;;
2164 (define_insn "mve_vcmpgeq_n_f<mode>"
2165   [
2166    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2167         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2168                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2169          VCMPGEQ_N_F))
2170   ]
2171   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2172   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2173   [(set_attr "type" "mve_move")
2174 ])
2175
2176 ;;
2177 ;; [vcmpgtq_f])
2178 ;;
2179 (define_insn "mve_vcmpgtq_f<mode>"
2180   [
2181    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2182         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2183                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2184          VCMPGTQ_F))
2185   ]
2186   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2187   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2188   [(set_attr "type" "mve_move")
2189 ])
2190
2191 ;;
2192 ;; [vcmpgtq_n_f])
2193 ;;
2194 (define_insn "mve_vcmpgtq_n_f<mode>"
2195   [
2196    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2197         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2198                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2199          VCMPGTQ_N_F))
2200   ]
2201   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2202   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2203   [(set_attr "type" "mve_move")
2204 ])
2205
2206 ;;
2207 ;; [vcmpleq_f])
2208 ;;
2209 (define_insn "mve_vcmpleq_f<mode>"
2210   [
2211    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2212         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2213                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2214          VCMPLEQ_F))
2215   ]
2216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2217   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2218   [(set_attr "type" "mve_move")
2219 ])
2220
2221 ;;
2222 ;; [vcmpleq_n_f])
2223 ;;
2224 (define_insn "mve_vcmpleq_n_f<mode>"
2225   [
2226    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2227         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2228                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2229          VCMPLEQ_N_F))
2230   ]
2231   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2232   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2233   [(set_attr "type" "mve_move")
2234 ])
2235
2236 ;;
2237 ;; [vcmpltq_f])
2238 ;;
2239 (define_insn "mve_vcmpltq_f<mode>"
2240   [
2241    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2242         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2243                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2244          VCMPLTQ_F))
2245   ]
2246   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2247   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2248   [(set_attr "type" "mve_move")
2249 ])
2250
2251 ;;
2252 ;; [vcmpltq_n_f])
2253 ;;
2254 (define_insn "mve_vcmpltq_n_f<mode>"
2255   [
2256    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2257         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2258                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2259          VCMPLTQ_N_F))
2260   ]
2261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2262   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2263   [(set_attr "type" "mve_move")
2264 ])
2265
2266 ;;
2267 ;; [vcmpneq_f])
2268 ;;
2269 (define_insn "mve_vcmpneq_f<mode>"
2270   [
2271    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2272         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2273                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2274          VCMPNEQ_F))
2275   ]
2276   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2277   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2278   [(set_attr "type" "mve_move")
2279 ])
2280
2281 ;;
2282 ;; [vcmpneq_n_f])
2283 ;;
2284 (define_insn "mve_vcmpneq_n_f<mode>"
2285   [
2286    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2287         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2288                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2289          VCMPNEQ_N_F))
2290   ]
2291   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2292   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2293   [(set_attr "type" "mve_move")
2294 ])
2295
2296 ;;
2297 ;; [vcmulq_f])
2298 ;;
2299 (define_insn "mve_vcmulq_f<mode>"
2300   [
2301    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2302         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2303                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2304          VCMULQ_F))
2305   ]
2306   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2307   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2308   [(set_attr "type" "mve_move")
2309 ])
2310
2311 ;;
2312 ;; [vcmulq_rot180_f])
2313 ;;
2314 (define_insn "mve_vcmulq_rot180_f<mode>"
2315   [
2316    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2317         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2318                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2319          VCMULQ_ROT180_F))
2320   ]
2321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2322   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2323   [(set_attr "type" "mve_move")
2324 ])
2325
2326 ;;
2327 ;; [vcmulq_rot270_f])
2328 ;;
2329 (define_insn "mve_vcmulq_rot270_f<mode>"
2330   [
2331    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2332         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2333                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2334          VCMULQ_ROT270_F))
2335   ]
2336   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2337   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2338   [(set_attr "type" "mve_move")
2339 ])
2340
2341 ;;
2342 ;; [vcmulq_rot90_f])
2343 ;;
2344 (define_insn "mve_vcmulq_rot90_f<mode>"
2345   [
2346    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2347         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2348                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2349          VCMULQ_ROT90_F))
2350   ]
2351   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2352   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2353   [(set_attr "type" "mve_move")
2354 ])
2355
2356 ;;
2357 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2358 ;;
2359 (define_insn "mve_vctp<mode1>q_mhi"
2360   [
2361    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2362         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2363                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2364          VCTPQ_M))
2365   ]
2366   "TARGET_HAVE_MVE"
2367   "vpst\;vctpt.<mode1> %1"
2368   [(set_attr "type" "mve_move")
2369    (set_attr "length""8")])
2370
2371 ;;
2372 ;; [vcvtbq_f16_f32])
2373 ;;
2374 (define_insn "mve_vcvtbq_f16_f32v8hf"
2375   [
2376    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2377         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2378                       (match_operand:V4SF 2 "s_register_operand" "w")]
2379          VCVTBQ_F16_F32))
2380   ]
2381   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2382   "vcvtb.f16.f32 %q0, %q2"
2383   [(set_attr "type" "mve_move")
2384 ])
2385
2386 ;;
2387 ;; [vcvttq_f16_f32])
2388 ;;
2389 (define_insn "mve_vcvttq_f16_f32v8hf"
2390   [
2391    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2392         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2393                       (match_operand:V4SF 2 "s_register_operand" "w")]
2394          VCVTTQ_F16_F32))
2395   ]
2396   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2397   "vcvtt.f16.f32 %q0, %q2"
2398   [(set_attr "type" "mve_move")
2399 ])
2400
2401 ;;
2402 ;; [veorq_f])
2403 ;;
2404 (define_insn "mve_veorq_f<mode>"
2405   [
2406    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2407         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2408                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2409          VEORQ_F))
2410   ]
2411   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2412   "veor %q0, %q1, %q2"
2413   [(set_attr "type" "mve_move")
2414 ])
2415
2416 ;;
2417 ;; [vmaxnmaq_f])
2418 ;;
2419 (define_insn "mve_vmaxnmaq_f<mode>"
2420   [
2421    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2422         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2423                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2424          VMAXNMAQ_F))
2425   ]
2426   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2427   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2428   [(set_attr "type" "mve_move")
2429 ])
2430
2431 ;;
2432 ;; [vmaxnmavq_f])
2433 ;;
2434 (define_insn "mve_vmaxnmavq_f<mode>"
2435   [
2436    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2437         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2438                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2439          VMAXNMAVQ_F))
2440   ]
2441   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2442   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2443   [(set_attr "type" "mve_move")
2444 ])
2445
2446 ;;
2447 ;; [vmaxnmq_f])
2448 ;;
2449 (define_insn "mve_vmaxnmq_f<mode>"
2450   [
2451    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2452         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2453                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2454   ]
2455   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2456   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2457   [(set_attr "type" "mve_move")
2458 ])
2459
2460 ;;
2461 ;; [vmaxnmvq_f])
2462 ;;
2463 (define_insn "mve_vmaxnmvq_f<mode>"
2464   [
2465    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2466         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2467                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2468          VMAXNMVQ_F))
2469   ]
2470   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2471   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2472   [(set_attr "type" "mve_move")
2473 ])
2474
2475 ;;
2476 ;; [vminnmaq_f])
2477 ;;
2478 (define_insn "mve_vminnmaq_f<mode>"
2479   [
2480    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2481         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2482                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2483          VMINNMAQ_F))
2484   ]
2485   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2486   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2487   [(set_attr "type" "mve_move")
2488 ])
2489
2490 ;;
2491 ;; [vminnmavq_f])
2492 ;;
2493 (define_insn "mve_vminnmavq_f<mode>"
2494   [
2495    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2496         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2497                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2498          VMINNMAVQ_F))
2499   ]
2500   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2501   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2502   [(set_attr "type" "mve_move")
2503 ])
2504
2505 ;;
2506 ;; [vminnmq_f])
2507 ;;
2508 (define_insn "mve_vminnmq_f<mode>"
2509   [
2510    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2511         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2512                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2513   ]
2514   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2515   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2516   [(set_attr "type" "mve_move")
2517 ])
2518
2519 ;;
2520 ;; [vminnmvq_f])
2521 ;;
2522 (define_insn "mve_vminnmvq_f<mode>"
2523   [
2524    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2525         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2526                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2527          VMINNMVQ_F))
2528   ]
2529   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2530   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2531   [(set_attr "type" "mve_move")
2532 ])
2533
2534 ;;
2535 ;; [vmlaldavq_u, vmlaldavq_s])
2536 ;;
2537 (define_insn "mve_vmlaldavq_<supf><mode>"
2538   [
2539    (set (match_operand:DI 0 "s_register_operand" "=r")
2540         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2541                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2542          VMLALDAVQ))
2543   ]
2544   "TARGET_HAVE_MVE"
2545   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2546   [(set_attr "type" "mve_move")
2547 ])
2548
2549 ;;
2550 ;; [vmlaldavxq_s])
2551 ;;
2552 (define_insn "mve_vmlaldavxq_s<mode>"
2553   [
2554    (set (match_operand:DI 0 "s_register_operand" "=r")
2555         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2556                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2557          VMLALDAVXQ_S))
2558   ]
2559   "TARGET_HAVE_MVE"
2560   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2561   [(set_attr "type" "mve_move")
2562 ])
2563
2564 ;;
2565 ;; [vmlsldavq_s])
2566 ;;
2567 (define_insn "mve_vmlsldavq_s<mode>"
2568   [
2569    (set (match_operand:DI 0 "s_register_operand" "=r")
2570         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2571                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2572          VMLSLDAVQ_S))
2573   ]
2574   "TARGET_HAVE_MVE"
2575   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2576   [(set_attr "type" "mve_move")
2577 ])
2578
2579 ;;
2580 ;; [vmlsldavxq_s])
2581 ;;
2582 (define_insn "mve_vmlsldavxq_s<mode>"
2583   [
2584    (set (match_operand:DI 0 "s_register_operand" "=r")
2585         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2586                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2587          VMLSLDAVXQ_S))
2588   ]
2589   "TARGET_HAVE_MVE"
2590   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2591   [(set_attr "type" "mve_move")
2592 ])
2593
2594 ;;
2595 ;; [vmovnbq_u, vmovnbq_s])
2596 ;;
2597 (define_insn "mve_vmovnbq_<supf><mode>"
2598   [
2599    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2600         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2601                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2602          VMOVNBQ))
2603   ]
2604   "TARGET_HAVE_MVE"
2605   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2606   [(set_attr "type" "mve_move")
2607 ])
2608
2609 ;;
2610 ;; [vmovntq_s, vmovntq_u])
2611 ;;
2612 (define_insn "mve_vmovntq_<supf><mode>"
2613   [
2614    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2615         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2616                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2617          VMOVNTQ))
2618   ]
2619   "TARGET_HAVE_MVE"
2620   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2621   [(set_attr "type" "mve_move")
2622 ])
2623
2624 ;;
2625 ;; [vmulq_f])
2626 ;;
2627 (define_insn "mve_vmulq_f<mode>"
2628   [
2629    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2630         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2631                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2632   ]
2633   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2634   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2635   [(set_attr "type" "mve_move")
2636 ])
2637
2638 ;;
2639 ;; [vmulq_n_f])
2640 ;;
2641 (define_insn "mve_vmulq_n_f<mode>"
2642   [
2643    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2644         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2645                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2646          VMULQ_N_F))
2647   ]
2648   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2649   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2650   [(set_attr "type" "mve_move")
2651 ])
2652
2653 ;;
2654 ;; [vornq_f])
2655 ;;
2656 (define_insn "mve_vornq_f<mode>"
2657   [
2658    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2659         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2660                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2661          VORNQ_F))
2662   ]
2663   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2664   "vorn %q0, %q1, %q2"
2665   [(set_attr "type" "mve_move")
2666 ])
2667
2668 ;;
2669 ;; [vorrq_f])
2670 ;;
2671 (define_insn "mve_vorrq_f<mode>"
2672   [
2673    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2674         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2675                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2676   ]
2677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2678   "vorr %q0, %q1, %q2"
2679   [(set_attr "type" "mve_move")
2680 ])
2681
2682 ;;
2683 ;; [vorrq_n_u, vorrq_n_s])
2684 ;;
2685 (define_insn "mve_vorrq_n_<supf><mode>"
2686   [
2687    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2688         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2689                        (match_operand:SI 2 "immediate_operand" "i")]
2690          VORRQ_N))
2691   ]
2692   "TARGET_HAVE_MVE"
2693   "vorr.i%#<V_sz_elem>  %q0, %2"
2694   [(set_attr "type" "mve_move")
2695 ])
2696
2697 ;;
2698 ;; [vqdmullbq_n_s])
2699 ;;
2700 (define_insn "mve_vqdmullbq_n_s<mode>"
2701   [
2702    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2703         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2704                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2705          VQDMULLBQ_N_S))
2706   ]
2707   "TARGET_HAVE_MVE"
2708   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2709   [(set_attr "type" "mve_move")
2710 ])
2711
2712 ;;
2713 ;; [vqdmullbq_s])
2714 ;;
2715 (define_insn "mve_vqdmullbq_s<mode>"
2716   [
2717    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2718         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2719                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2720          VQDMULLBQ_S))
2721   ]
2722   "TARGET_HAVE_MVE"
2723   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2724   [(set_attr "type" "mve_move")
2725 ])
2726
2727 ;;
2728 ;; [vqdmulltq_n_s])
2729 ;;
2730 (define_insn "mve_vqdmulltq_n_s<mode>"
2731   [
2732    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2733         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2734                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2735          VQDMULLTQ_N_S))
2736   ]
2737   "TARGET_HAVE_MVE"
2738   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2739   [(set_attr "type" "mve_move")
2740 ])
2741
2742 ;;
2743 ;; [vqdmulltq_s])
2744 ;;
2745 (define_insn "mve_vqdmulltq_s<mode>"
2746   [
2747    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2748         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2749                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2750          VQDMULLTQ_S))
2751   ]
2752   "TARGET_HAVE_MVE"
2753   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2754   [(set_attr "type" "mve_move")
2755 ])
2756
2757 ;;
2758 ;; [vqmovnbq_u, vqmovnbq_s])
2759 ;;
2760 (define_insn "mve_vqmovnbq_<supf><mode>"
2761   [
2762    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2763         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2764                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2765          VQMOVNBQ))
2766   ]
2767   "TARGET_HAVE_MVE"
2768   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2769   [(set_attr "type" "mve_move")
2770 ])
2771
2772 ;;
2773 ;; [vqmovntq_u, vqmovntq_s])
2774 ;;
2775 (define_insn "mve_vqmovntq_<supf><mode>"
2776   [
2777    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2778         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2779                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2780          VQMOVNTQ))
2781   ]
2782   "TARGET_HAVE_MVE"
2783   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2784   [(set_attr "type" "mve_move")
2785 ])
2786
2787 ;;
2788 ;; [vqmovunbq_s])
2789 ;;
2790 (define_insn "mve_vqmovunbq_s<mode>"
2791   [
2792    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2793         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2794                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2795          VQMOVUNBQ_S))
2796   ]
2797   "TARGET_HAVE_MVE"
2798   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2799   [(set_attr "type" "mve_move")
2800 ])
2801
2802 ;;
2803 ;; [vqmovuntq_s])
2804 ;;
2805 (define_insn "mve_vqmovuntq_s<mode>"
2806   [
2807    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2808         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2809                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2810          VQMOVUNTQ_S))
2811   ]
2812   "TARGET_HAVE_MVE"
2813   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2814   [(set_attr "type" "mve_move")
2815 ])
2816
2817 ;;
2818 ;; [vrmlaldavhxq_s])
2819 ;;
2820 (define_insn "mve_vrmlaldavhxq_sv4si"
2821   [
2822    (set (match_operand:DI 0 "s_register_operand" "=r")
2823         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2824                     (match_operand:V4SI 2 "s_register_operand" "w")]
2825          VRMLALDAVHXQ_S))
2826   ]
2827   "TARGET_HAVE_MVE"
2828   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2829   [(set_attr "type" "mve_move")
2830 ])
2831
2832 ;;
2833 ;; [vrmlsldavhq_s])
2834 ;;
2835 (define_insn "mve_vrmlsldavhq_sv4si"
2836   [
2837    (set (match_operand:DI 0 "s_register_operand" "=r")
2838         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2839                     (match_operand:V4SI 2 "s_register_operand" "w")]
2840          VRMLSLDAVHQ_S))
2841   ]
2842   "TARGET_HAVE_MVE"
2843   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2844   [(set_attr "type" "mve_move")
2845 ])
2846
2847 ;;
2848 ;; [vrmlsldavhxq_s])
2849 ;;
2850 (define_insn "mve_vrmlsldavhxq_sv4si"
2851   [
2852    (set (match_operand:DI 0 "s_register_operand" "=r")
2853         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2854                     (match_operand:V4SI 2 "s_register_operand" "w")]
2855          VRMLSLDAVHXQ_S))
2856   ]
2857   "TARGET_HAVE_MVE"
2858   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2859   [(set_attr "type" "mve_move")
2860 ])
2861
2862 ;;
2863 ;; [vshllbq_n_s, vshllbq_n_u])
2864 ;;
2865 (define_insn "mve_vshllbq_n_<supf><mode>"
2866   [
2867    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2868         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2869                                   (match_operand:SI 2 "immediate_operand" "i")]
2870          VSHLLBQ_N))
2871   ]
2872   "TARGET_HAVE_MVE"
2873   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2874   [(set_attr "type" "mve_move")
2875 ])
2876
2877 ;;
2878 ;; [vshlltq_n_u, vshlltq_n_s])
2879 ;;
2880 (define_insn "mve_vshlltq_n_<supf><mode>"
2881   [
2882    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2883         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2884                                   (match_operand:SI 2 "immediate_operand" "i")]
2885          VSHLLTQ_N))
2886   ]
2887   "TARGET_HAVE_MVE"
2888   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2889   [(set_attr "type" "mve_move")
2890 ])
2891
2892 ;;
2893 ;; [vsubq_f])
2894 ;;
2895 (define_insn "mve_vsubq_f<mode>"
2896   [
2897    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2898         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2899                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2900   ]
2901   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2902   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2903   [(set_attr "type" "mve_move")
2904 ])
2905
2906 ;;
2907 ;; [vmulltq_poly_p])
2908 ;;
2909 (define_insn "mve_vmulltq_poly_p<mode>"
2910   [
2911    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2912         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2913                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2914          VMULLTQ_POLY_P))
2915   ]
2916   "TARGET_HAVE_MVE"
2917   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2918   [(set_attr "type" "mve_move")
2919 ])
2920
2921 ;;
2922 ;; [vmullbq_poly_p])
2923 ;;
2924 (define_insn "mve_vmullbq_poly_p<mode>"
2925   [
2926    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2927         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2928                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2929          VMULLBQ_POLY_P))
2930   ]
2931   "TARGET_HAVE_MVE"
2932   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2933   [(set_attr "type" "mve_move")
2934 ])
2935
2936 ;;
2937 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2938 ;;
2939 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2940   [
2941    (set (match_operand:DI 0 "s_register_operand" "=r")
2942         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2943                     (match_operand:V4SI 2 "s_register_operand" "w")]
2944          VRMLALDAVHQ))
2945   ]
2946   "TARGET_HAVE_MVE"
2947   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2948   [(set_attr "type" "mve_move")
2949 ])
2950
2951 ;;
2952 ;; [vbicq_m_n_s, vbicq_m_n_u])
2953 ;;
2954 (define_insn "mve_vbicq_m_n_<supf><mode>"
2955   [
2956    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2957         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2958                        (match_operand:SI 2 "immediate_operand" "i")
2959                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2960          VBICQ_M_N))
2961   ]
2962   "TARGET_HAVE_MVE"
2963   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2964   [(set_attr "type" "mve_move")
2965    (set_attr "length""8")])
2966 ;;
2967 ;; [vcmpeqq_m_f])
2968 ;;
2969 (define_insn "mve_vcmpeqq_m_f<mode>"
2970   [
2971    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2972         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2973                     (match_operand:MVE_0 2 "s_register_operand" "w")
2974                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2975          VCMPEQQ_M_F))
2976   ]
2977   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2978   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2979   [(set_attr "type" "mve_move")
2980    (set_attr "length""8")])
2981 ;;
2982 ;; [vcvtaq_m_u, vcvtaq_m_s])
2983 ;;
2984 (define_insn "mve_vcvtaq_m_<supf><mode>"
2985   [
2986    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2987         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2988                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2989                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2990          VCVTAQ_M))
2991   ]
2992   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2993   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2994   [(set_attr "type" "mve_move")
2995    (set_attr "length""8")])
2996 ;;
2997 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2998 ;;
2999 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3000   [
3001    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3002         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3003                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3004                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3005          VCVTQ_M_TO_F))
3006   ]
3007   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3008   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3009   [(set_attr "type" "mve_move")
3010    (set_attr "length""8")])
3011 ;;
3012 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3013 ;;
3014 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3015   [
3016    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3017         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3018                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3019                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3020          VQRSHRNBQ_N))
3021   ]
3022   "TARGET_HAVE_MVE"
3023   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3024   [(set_attr "type" "mve_move")
3025 ])
3026 ;;
3027 ;; [vqrshrunbq_n_s])
3028 ;;
3029 (define_insn "mve_vqrshrunbq_n_s<mode>"
3030   [
3031    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3032         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3033                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3034                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3035          VQRSHRUNBQ_N_S))
3036   ]
3037   "TARGET_HAVE_MVE"
3038   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3039   [(set_attr "type" "mve_move")
3040 ])
3041 ;;
3042 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3043 ;;
3044 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3045   [
3046    (set (match_operand:DI 0 "s_register_operand" "=r")
3047         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3048                     (match_operand:V4SI 2 "s_register_operand" "w")
3049                     (match_operand:V4SI 3 "s_register_operand" "w")]
3050          VRMLALDAVHAQ))
3051   ]
3052   "TARGET_HAVE_MVE"
3053   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3054   [(set_attr "type" "mve_move")
3055 ])
3056
3057 ;;
3058 ;; [vabavq_s, vabavq_u])
3059 ;;
3060 (define_insn "mve_vabavq_<supf><mode>"
3061   [
3062    (set (match_operand:SI 0 "s_register_operand" "=r")
3063         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3064                     (match_operand:MVE_2 2 "s_register_operand" "w")
3065                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3066          VABAVQ))
3067   ]
3068   "TARGET_HAVE_MVE"
3069   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3070   [(set_attr "type" "mve_move")
3071 ])
3072
3073 ;;
3074 ;; [vshlcq_u vshlcq_s]
3075 ;;
3076 (define_expand "mve_vshlcq_vec_<supf><mode>"
3077  [(match_operand:MVE_2 0 "s_register_operand")
3078   (match_operand:MVE_2 1 "s_register_operand")
3079   (match_operand:SI 2 "s_register_operand")
3080   (match_operand:SI 3 "mve_imm_32")
3081   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3082  "TARGET_HAVE_MVE"
3083 {
3084   rtx ignore_wb = gen_reg_rtx (SImode);
3085   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3086                                       operands[2], operands[3]));
3087   DONE;
3088 })
3089
3090 (define_expand "mve_vshlcq_carry_<supf><mode>"
3091  [(match_operand:SI 0 "s_register_operand")
3092   (match_operand:MVE_2 1 "s_register_operand")
3093   (match_operand:SI 2 "s_register_operand")
3094   (match_operand:SI 3 "mve_imm_32")
3095   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3096  "TARGET_HAVE_MVE"
3097 {
3098   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3099   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3100                                       operands[2], operands[3]));
3101   DONE;
3102 })
3103
3104 (define_insn "mve_vshlcq_<supf><mode>"
3105  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3106        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3107                       (match_operand:SI 3 "s_register_operand" "1")
3108                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3109         VSHLCQ))
3110   (set (match_operand:SI  1 "s_register_operand" "=r")
3111        (unspec:SI [(match_dup 2)
3112                    (match_dup 3)
3113                    (match_dup 4)]
3114         VSHLCQ))]
3115  "TARGET_HAVE_MVE"
3116  "vshlc %q0, %1, %4")
3117
3118 ;;
3119 ;; [vabsq_m_s])
3120 ;;
3121 (define_insn "mve_vabsq_m_s<mode>"
3122   [
3123    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3124         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3125                        (match_operand:MVE_2 2 "s_register_operand" "w")
3126                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3127          VABSQ_M_S))
3128   ]
3129   "TARGET_HAVE_MVE"
3130   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3131   [(set_attr "type" "mve_move")
3132    (set_attr "length""8")])
3133
3134 ;;
3135 ;; [vaddvaq_p_u, vaddvaq_p_s])
3136 ;;
3137 (define_insn "mve_vaddvaq_p_<supf><mode>"
3138   [
3139    (set (match_operand:SI 0 "s_register_operand" "=Te")
3140         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3141                        (match_operand:MVE_2 2 "s_register_operand" "w")
3142                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3143          VADDVAQ_P))
3144   ]
3145   "TARGET_HAVE_MVE"
3146   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3147   [(set_attr "type" "mve_move")
3148    (set_attr "length""8")])
3149
3150 ;;
3151 ;; [vclsq_m_s])
3152 ;;
3153 (define_insn "mve_vclsq_m_s<mode>"
3154   [
3155    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3156         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3157                        (match_operand:MVE_2 2 "s_register_operand" "w")
3158                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3159          VCLSQ_M_S))
3160   ]
3161   "TARGET_HAVE_MVE"
3162   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3163   [(set_attr "type" "mve_move")
3164    (set_attr "length""8")])
3165
3166 ;;
3167 ;; [vclzq_m_s, vclzq_m_u])
3168 ;;
3169 (define_insn "mve_vclzq_m_<supf><mode>"
3170   [
3171    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3172         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3173                        (match_operand:MVE_2 2 "s_register_operand" "w")
3174                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3175          VCLZQ_M))
3176   ]
3177   "TARGET_HAVE_MVE"
3178   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3179   [(set_attr "type" "mve_move")
3180    (set_attr "length""8")])
3181
3182 ;;
3183 ;; [vcmpcsq_m_n_u])
3184 ;;
3185 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3186   [
3187    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3188         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3189                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3190                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3191          VCMPCSQ_M_N_U))
3192   ]
3193   "TARGET_HAVE_MVE"
3194   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3195   [(set_attr "type" "mve_move")
3196    (set_attr "length""8")])
3197
3198 ;;
3199 ;; [vcmpcsq_m_u])
3200 ;;
3201 (define_insn "mve_vcmpcsq_m_u<mode>"
3202   [
3203    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3204         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3205                        (match_operand:MVE_2 2 "s_register_operand" "w")
3206                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3207          VCMPCSQ_M_U))
3208   ]
3209   "TARGET_HAVE_MVE"
3210   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3211   [(set_attr "type" "mve_move")
3212    (set_attr "length""8")])
3213
3214 ;;
3215 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3216 ;;
3217 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3218   [
3219    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3220         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3221                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3222                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3223          VCMPEQQ_M_N))
3224   ]
3225   "TARGET_HAVE_MVE"
3226   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3227   [(set_attr "type" "mve_move")
3228    (set_attr "length""8")])
3229
3230 ;;
3231 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3232 ;;
3233 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3234   [
3235    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3236         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3237                        (match_operand:MVE_2 2 "s_register_operand" "w")
3238                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3239          VCMPEQQ_M))
3240   ]
3241   "TARGET_HAVE_MVE"
3242   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3243   [(set_attr "type" "mve_move")
3244    (set_attr "length""8")])
3245
3246 ;;
3247 ;; [vcmpgeq_m_n_s])
3248 ;;
3249 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3250   [
3251    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3252         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3253                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3254                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3255          VCMPGEQ_M_N_S))
3256   ]
3257   "TARGET_HAVE_MVE"
3258   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3259   [(set_attr "type" "mve_move")
3260    (set_attr "length""8")])
3261
3262 ;;
3263 ;; [vcmpgeq_m_s])
3264 ;;
3265 (define_insn "mve_vcmpgeq_m_s<mode>"
3266   [
3267    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3268         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3269                        (match_operand:MVE_2 2 "s_register_operand" "w")
3270                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3271          VCMPGEQ_M_S))
3272   ]
3273   "TARGET_HAVE_MVE"
3274   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3275   [(set_attr "type" "mve_move")
3276    (set_attr "length""8")])
3277
3278 ;;
3279 ;; [vcmpgtq_m_n_s])
3280 ;;
3281 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3282   [
3283    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3284         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3285                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3286                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3287          VCMPGTQ_M_N_S))
3288   ]
3289   "TARGET_HAVE_MVE"
3290   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3291   [(set_attr "type" "mve_move")
3292    (set_attr "length""8")])
3293
3294 ;;
3295 ;; [vcmpgtq_m_s])
3296 ;;
3297 (define_insn "mve_vcmpgtq_m_s<mode>"
3298   [
3299    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3300         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3301                        (match_operand:MVE_2 2 "s_register_operand" "w")
3302                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3303          VCMPGTQ_M_S))
3304   ]
3305   "TARGET_HAVE_MVE"
3306   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3307   [(set_attr "type" "mve_move")
3308    (set_attr "length""8")])
3309
3310 ;;
3311 ;; [vcmphiq_m_n_u])
3312 ;;
3313 (define_insn "mve_vcmphiq_m_n_u<mode>"
3314   [
3315    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3316         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3317                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3318                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3319          VCMPHIQ_M_N_U))
3320   ]
3321   "TARGET_HAVE_MVE"
3322   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3323   [(set_attr "type" "mve_move")
3324    (set_attr "length""8")])
3325
3326 ;;
3327 ;; [vcmphiq_m_u])
3328 ;;
3329 (define_insn "mve_vcmphiq_m_u<mode>"
3330   [
3331    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3332         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3333                        (match_operand:MVE_2 2 "s_register_operand" "w")
3334                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3335          VCMPHIQ_M_U))
3336   ]
3337   "TARGET_HAVE_MVE"
3338   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3339   [(set_attr "type" "mve_move")
3340    (set_attr "length""8")])
3341
3342 ;;
3343 ;; [vcmpleq_m_n_s])
3344 ;;
3345 (define_insn "mve_vcmpleq_m_n_s<mode>"
3346   [
3347    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3348         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3349                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3350                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3351          VCMPLEQ_M_N_S))
3352   ]
3353   "TARGET_HAVE_MVE"
3354   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3355   [(set_attr "type" "mve_move")
3356    (set_attr "length""8")])
3357
3358 ;;
3359 ;; [vcmpleq_m_s])
3360 ;;
3361 (define_insn "mve_vcmpleq_m_s<mode>"
3362   [
3363    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3364         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3365                        (match_operand:MVE_2 2 "s_register_operand" "w")
3366                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3367          VCMPLEQ_M_S))
3368   ]
3369   "TARGET_HAVE_MVE"
3370   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3371   [(set_attr "type" "mve_move")
3372    (set_attr "length""8")])
3373
3374 ;;
3375 ;; [vcmpltq_m_n_s])
3376 ;;
3377 (define_insn "mve_vcmpltq_m_n_s<mode>"
3378   [
3379    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3380         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3381                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3382                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3383          VCMPLTQ_M_N_S))
3384   ]
3385   "TARGET_HAVE_MVE"
3386   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3387   [(set_attr "type" "mve_move")
3388    (set_attr "length""8")])
3389
3390 ;;
3391 ;; [vcmpltq_m_s])
3392 ;;
3393 (define_insn "mve_vcmpltq_m_s<mode>"
3394   [
3395    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3396         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3397                        (match_operand:MVE_2 2 "s_register_operand" "w")
3398                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3399          VCMPLTQ_M_S))
3400   ]
3401   "TARGET_HAVE_MVE"
3402   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3403   [(set_attr "type" "mve_move")
3404    (set_attr "length""8")])
3405
3406 ;;
3407 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3408 ;;
3409 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3410   [
3411    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3412         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3413                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3414                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3415          VCMPNEQ_M_N))
3416   ]
3417   "TARGET_HAVE_MVE"
3418   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3419   [(set_attr "type" "mve_move")
3420    (set_attr "length""8")])
3421
3422 ;;
3423 ;; [vcmpneq_m_s, vcmpneq_m_u])
3424 ;;
3425 (define_insn "mve_vcmpneq_m_<supf><mode>"
3426   [
3427    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3428         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3429                        (match_operand:MVE_2 2 "s_register_operand" "w")
3430                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3431          VCMPNEQ_M))
3432   ]
3433   "TARGET_HAVE_MVE"
3434   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3435   [(set_attr "type" "mve_move")
3436    (set_attr "length""8")])
3437
3438 ;;
3439 ;; [vdupq_m_n_s, vdupq_m_n_u])
3440 ;;
3441 (define_insn "mve_vdupq_m_n_<supf><mode>"
3442   [
3443    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3444         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3445                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3446                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3447          VDUPQ_M_N))
3448   ]
3449   "TARGET_HAVE_MVE"
3450   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3451   [(set_attr "type" "mve_move")
3452    (set_attr "length""8")])
3453
3454 ;;
3455 ;; [vmaxaq_m_s])
3456 ;;
3457 (define_insn "mve_vmaxaq_m_s<mode>"
3458   [
3459    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3460         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3461                        (match_operand:MVE_2 2 "s_register_operand" "w")
3462                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3463          VMAXAQ_M_S))
3464   ]
3465   "TARGET_HAVE_MVE"
3466   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3467   [(set_attr "type" "mve_move")
3468    (set_attr "length""8")])
3469
3470 ;;
3471 ;; [vmaxavq_p_s])
3472 ;;
3473 (define_insn "mve_vmaxavq_p_s<mode>"
3474   [
3475    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3476         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3477                        (match_operand:MVE_2 2 "s_register_operand" "w")
3478                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3479          VMAXAVQ_P_S))
3480   ]
3481   "TARGET_HAVE_MVE"
3482   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3483   [(set_attr "type" "mve_move")
3484    (set_attr "length""8")])
3485
3486 ;;
3487 ;; [vmaxvq_p_u, vmaxvq_p_s])
3488 ;;
3489 (define_insn "mve_vmaxvq_p_<supf><mode>"
3490   [
3491    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3492         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3493                        (match_operand:MVE_2 2 "s_register_operand" "w")
3494                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3495          VMAXVQ_P))
3496   ]
3497   "TARGET_HAVE_MVE"
3498   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3499   [(set_attr "type" "mve_move")
3500    (set_attr "length""8")])
3501
3502 ;;
3503 ;; [vminaq_m_s])
3504 ;;
3505 (define_insn "mve_vminaq_m_s<mode>"
3506   [
3507    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3508         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3509                        (match_operand:MVE_2 2 "s_register_operand" "w")
3510                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3511          VMINAQ_M_S))
3512   ]
3513   "TARGET_HAVE_MVE"
3514   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3515   [(set_attr "type" "mve_move")
3516    (set_attr "length""8")])
3517
3518 ;;
3519 ;; [vminavq_p_s])
3520 ;;
3521 (define_insn "mve_vminavq_p_s<mode>"
3522   [
3523    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3524         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3525                        (match_operand:MVE_2 2 "s_register_operand" "w")
3526                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3527          VMINAVQ_P_S))
3528   ]
3529   "TARGET_HAVE_MVE"
3530   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3531   [(set_attr "type" "mve_move")
3532    (set_attr "length""8")])
3533
3534 ;;
3535 ;; [vminvq_p_s, vminvq_p_u])
3536 ;;
3537 (define_insn "mve_vminvq_p_<supf><mode>"
3538   [
3539    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3540         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3541                        (match_operand:MVE_2 2 "s_register_operand" "w")
3542                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3543          VMINVQ_P))
3544   ]
3545   "TARGET_HAVE_MVE"
3546   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3547   [(set_attr "type" "mve_move")
3548    (set_attr "length""8")])
3549
3550 ;;
3551 ;; [vmladavaq_u, vmladavaq_s])
3552 ;;
3553 (define_insn "mve_vmladavaq_<supf><mode>"
3554   [
3555    (set (match_operand:SI 0 "s_register_operand" "=Te")
3556         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3557                        (match_operand:MVE_2 2 "s_register_operand" "w")
3558                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3559          VMLADAVAQ))
3560   ]
3561   "TARGET_HAVE_MVE"
3562   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3563   [(set_attr "type" "mve_move")
3564 ])
3565
3566 ;;
3567 ;; [vmladavq_p_u, vmladavq_p_s])
3568 ;;
3569 (define_insn "mve_vmladavq_p_<supf><mode>"
3570   [
3571    (set (match_operand:SI 0 "s_register_operand" "=Te")
3572         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3573                        (match_operand:MVE_2 2 "s_register_operand" "w")
3574                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3575          VMLADAVQ_P))
3576   ]
3577   "TARGET_HAVE_MVE"
3578   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3579   [(set_attr "type" "mve_move")
3580    (set_attr "length""8")])
3581
3582 ;;
3583 ;; [vmladavxq_p_s])
3584 ;;
3585 (define_insn "mve_vmladavxq_p_s<mode>"
3586   [
3587    (set (match_operand:SI 0 "s_register_operand" "=Te")
3588         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3589                        (match_operand:MVE_2 2 "s_register_operand" "w")
3590                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3591          VMLADAVXQ_P_S))
3592   ]
3593   "TARGET_HAVE_MVE"
3594   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3595   [(set_attr "type" "mve_move")
3596    (set_attr "length""8")])
3597
3598 ;;
3599 ;; [vmlaq_n_u, vmlaq_n_s])
3600 ;;
3601 (define_insn "mve_vmlaq_n_<supf><mode>"
3602   [
3603    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3604         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3605                        (match_operand:MVE_2 2 "s_register_operand" "w")
3606                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3607          VMLAQ_N))
3608   ]
3609   "TARGET_HAVE_MVE"
3610   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3611   [(set_attr "type" "mve_move")
3612 ])
3613
3614 ;;
3615 ;; [vmlasq_n_u, vmlasq_n_s])
3616 ;;
3617 (define_insn "mve_vmlasq_n_<supf><mode>"
3618   [
3619    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3620         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3621                        (match_operand:MVE_2 2 "s_register_operand" "w")
3622                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3623          VMLASQ_N))
3624   ]
3625   "TARGET_HAVE_MVE"
3626   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3627   [(set_attr "type" "mve_move")
3628 ])
3629
3630 ;;
3631 ;; [vmlsdavq_p_s])
3632 ;;
3633 (define_insn "mve_vmlsdavq_p_s<mode>"
3634   [
3635    (set (match_operand:SI 0 "s_register_operand" "=Te")
3636         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3637                        (match_operand:MVE_2 2 "s_register_operand" "w")
3638                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3639          VMLSDAVQ_P_S))
3640   ]
3641   "TARGET_HAVE_MVE"
3642   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3643   [(set_attr "type" "mve_move")
3644    (set_attr "length""8")])
3645
3646 ;;
3647 ;; [vmlsdavxq_p_s])
3648 ;;
3649 (define_insn "mve_vmlsdavxq_p_s<mode>"
3650   [
3651    (set (match_operand:SI 0 "s_register_operand" "=Te")
3652         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3653                        (match_operand:MVE_2 2 "s_register_operand" "w")
3654                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3655          VMLSDAVXQ_P_S))
3656   ]
3657   "TARGET_HAVE_MVE"
3658   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3659   [(set_attr "type" "mve_move")
3660    (set_attr "length""8")])
3661
3662 ;;
3663 ;; [vmvnq_m_s, vmvnq_m_u])
3664 ;;
3665 (define_insn "mve_vmvnq_m_<supf><mode>"
3666   [
3667    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3668         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3669                        (match_operand:MVE_2 2 "s_register_operand" "w")
3670                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3671          VMVNQ_M))
3672   ]
3673   "TARGET_HAVE_MVE"
3674   "vpst\;vmvnt %q0, %q2"
3675   [(set_attr "type" "mve_move")
3676    (set_attr "length""8")])
3677
3678 ;;
3679 ;; [vnegq_m_s])
3680 ;;
3681 (define_insn "mve_vnegq_m_s<mode>"
3682   [
3683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3685                        (match_operand:MVE_2 2 "s_register_operand" "w")
3686                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3687          VNEGQ_M_S))
3688   ]
3689   "TARGET_HAVE_MVE"
3690   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3691   [(set_attr "type" "mve_move")
3692    (set_attr "length""8")])
3693
3694 ;;
3695 ;; [vpselq_u, vpselq_s])
3696 ;;
3697 (define_insn "mve_vpselq_<supf><mode>"
3698   [
3699    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3700         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3701                        (match_operand:MVE_1 2 "s_register_operand" "w")
3702                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3703          VPSELQ))
3704   ]
3705   "TARGET_HAVE_MVE"
3706   "vpsel %q0, %q1, %q2"
3707   [(set_attr "type" "mve_move")
3708 ])
3709
3710 ;;
3711 ;; [vqabsq_m_s])
3712 ;;
3713 (define_insn "mve_vqabsq_m_s<mode>"
3714   [
3715    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3716         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3717                        (match_operand:MVE_2 2 "s_register_operand" "w")
3718                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3719          VQABSQ_M_S))
3720   ]
3721   "TARGET_HAVE_MVE"
3722   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3723   [(set_attr "type" "mve_move")
3724    (set_attr "length""8")])
3725
3726 ;;
3727 ;; [vqdmlahq_n_s])
3728 ;;
3729 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3730   [
3731    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3732         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3733                        (match_operand:MVE_2 2 "s_register_operand" "w")
3734                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3735          VQDMLAHQ_N))
3736   ]
3737   "TARGET_HAVE_MVE"
3738   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3739   [(set_attr "type" "mve_move")
3740 ])
3741
3742 ;;
3743 ;; [vqdmlashq_n_s])
3744 ;;
3745 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3746   [
3747    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3748         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3749                        (match_operand:MVE_2 2 "s_register_operand" "w")
3750                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3751          VQDMLASHQ_N))
3752   ]
3753   "TARGET_HAVE_MVE"
3754   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3755   [(set_attr "type" "mve_move")
3756 ])
3757
3758 ;;
3759 ;; [vqnegq_m_s])
3760 ;;
3761 (define_insn "mve_vqnegq_m_s<mode>"
3762   [
3763    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3764         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3765                        (match_operand:MVE_2 2 "s_register_operand" "w")
3766                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3767          VQNEGQ_M_S))
3768   ]
3769   "TARGET_HAVE_MVE"
3770   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3771   [(set_attr "type" "mve_move")
3772    (set_attr "length""8")])
3773
3774 ;;
3775 ;; [vqrdmladhq_s])
3776 ;;
3777 (define_insn "mve_vqrdmladhq_s<mode>"
3778   [
3779    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3780         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3781                        (match_operand:MVE_2 2 "s_register_operand" "w")
3782                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3783          VQRDMLADHQ_S))
3784   ]
3785   "TARGET_HAVE_MVE"
3786   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3787   [(set_attr "type" "mve_move")
3788 ])
3789
3790 ;;
3791 ;; [vqrdmladhxq_s])
3792 ;;
3793 (define_insn "mve_vqrdmladhxq_s<mode>"
3794   [
3795    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3796         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3797                        (match_operand:MVE_2 2 "s_register_operand" "w")
3798                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3799          VQRDMLADHXQ_S))
3800   ]
3801   "TARGET_HAVE_MVE"
3802   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3803   [(set_attr "type" "mve_move")
3804 ])
3805
3806 ;;
3807 ;; [vqrdmlahq_n_s])
3808 ;;
3809 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3810   [
3811    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3812         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3813                        (match_operand:MVE_2 2 "s_register_operand" "w")
3814                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3815          VQRDMLAHQ_N))
3816   ]
3817   "TARGET_HAVE_MVE"
3818   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3819   [(set_attr "type" "mve_move")
3820 ])
3821
3822 ;;
3823 ;; [vqrdmlashq_n_s])
3824 ;;
3825 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3826   [
3827    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3828         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3829                        (match_operand:MVE_2 2 "s_register_operand" "w")
3830                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3831          VQRDMLASHQ_N))
3832   ]
3833   "TARGET_HAVE_MVE"
3834   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3835   [(set_attr "type" "mve_move")
3836 ])
3837
3838 ;;
3839 ;; [vqrdmlsdhq_s])
3840 ;;
3841 (define_insn "mve_vqrdmlsdhq_s<mode>"
3842   [
3843    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3844         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3845                        (match_operand:MVE_2 2 "s_register_operand" "w")
3846                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3847          VQRDMLSDHQ_S))
3848   ]
3849   "TARGET_HAVE_MVE"
3850   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3851   [(set_attr "type" "mve_move")
3852 ])
3853
3854 ;;
3855 ;; [vqrdmlsdhxq_s])
3856 ;;
3857 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3858   [
3859    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3860         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3861                        (match_operand:MVE_2 2 "s_register_operand" "w")
3862                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3863          VQRDMLSDHXQ_S))
3864   ]
3865   "TARGET_HAVE_MVE"
3866   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3867   [(set_attr "type" "mve_move")
3868 ])
3869
3870 ;;
3871 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3872 ;;
3873 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3874   [
3875    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3876         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3877                        (match_operand:SI 2 "s_register_operand" "r")
3878                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3879          VQRSHLQ_M_N))
3880   ]
3881   "TARGET_HAVE_MVE"
3882   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3883   [(set_attr "type" "mve_move")
3884    (set_attr "length""8")])
3885
3886 ;;
3887 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3888 ;;
3889 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3890   [
3891    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3892         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3893                        (match_operand:SI 2 "s_register_operand" "r")
3894                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3895          VQSHLQ_M_R))
3896   ]
3897   "TARGET_HAVE_MVE"
3898   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3899   [(set_attr "type" "mve_move")
3900    (set_attr "length""8")])
3901
3902 ;;
3903 ;; [vrev64q_m_u, vrev64q_m_s])
3904 ;;
3905 (define_insn "mve_vrev64q_m_<supf><mode>"
3906   [
3907    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3908         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3909                        (match_operand:MVE_2 2 "s_register_operand" "w")
3910                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3911          VREV64Q_M))
3912   ]
3913   "TARGET_HAVE_MVE"
3914   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3915   [(set_attr "type" "mve_move")
3916    (set_attr "length""8")])
3917
3918 ;;
3919 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3920 ;;
3921 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3922   [
3923    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3924         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3925                        (match_operand:SI 2 "s_register_operand" "r")
3926                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3927          VRSHLQ_M_N))
3928   ]
3929   "TARGET_HAVE_MVE"
3930   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3931   [(set_attr "type" "mve_move")
3932    (set_attr "length""8")])
3933
3934 ;;
3935 ;; [vshlq_m_r_u, vshlq_m_r_s])
3936 ;;
3937 (define_insn "mve_vshlq_m_r_<supf><mode>"
3938   [
3939    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3940         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3941                        (match_operand:SI 2 "s_register_operand" "r")
3942                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3943          VSHLQ_M_R))
3944   ]
3945   "TARGET_HAVE_MVE"
3946   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3947   [(set_attr "type" "mve_move")
3948    (set_attr "length""8")])
3949
3950 ;;
3951 ;; [vsliq_n_u, vsliq_n_s])
3952 ;;
3953 (define_insn "mve_vsliq_n_<supf><mode>"
3954   [
3955    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3956         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3957                        (match_operand:MVE_2 2 "s_register_operand" "w")
3958                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3959          VSLIQ_N))
3960   ]
3961   "TARGET_HAVE_MVE"
3962   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3963   [(set_attr "type" "mve_move")
3964 ])
3965
3966 ;;
3967 ;; [vsriq_n_u, vsriq_n_s])
3968 ;;
3969 (define_insn "mve_vsriq_n_<supf><mode>"
3970   [
3971    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3972         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3973                        (match_operand:MVE_2 2 "s_register_operand" "w")
3974                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3975          VSRIQ_N))
3976   ]
3977   "TARGET_HAVE_MVE"
3978   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3979   [(set_attr "type" "mve_move")
3980 ])
3981
3982 ;;
3983 ;; [vqdmlsdhxq_s])
3984 ;;
3985 (define_insn "mve_vqdmlsdhxq_s<mode>"
3986   [
3987    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3988         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3989                        (match_operand:MVE_2 2 "s_register_operand" "w")
3990                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3991          VQDMLSDHXQ_S))
3992   ]
3993   "TARGET_HAVE_MVE"
3994   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3995   [(set_attr "type" "mve_move")
3996 ])
3997
3998 ;;
3999 ;; [vqdmlsdhq_s])
4000 ;;
4001 (define_insn "mve_vqdmlsdhq_s<mode>"
4002   [
4003    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4004         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4005                        (match_operand:MVE_2 2 "s_register_operand" "w")
4006                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4007          VQDMLSDHQ_S))
4008   ]
4009   "TARGET_HAVE_MVE"
4010   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4011   [(set_attr "type" "mve_move")
4012 ])
4013
4014 ;;
4015 ;; [vqdmladhxq_s])
4016 ;;
4017 (define_insn "mve_vqdmladhxq_s<mode>"
4018   [
4019    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4020         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4021                        (match_operand:MVE_2 2 "s_register_operand" "w")
4022                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4023          VQDMLADHXQ_S))
4024   ]
4025   "TARGET_HAVE_MVE"
4026   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4027   [(set_attr "type" "mve_move")
4028 ])
4029
4030 ;;
4031 ;; [vqdmladhq_s])
4032 ;;
4033 (define_insn "mve_vqdmladhq_s<mode>"
4034   [
4035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4037                        (match_operand:MVE_2 2 "s_register_operand" "w")
4038                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4039          VQDMLADHQ_S))
4040   ]
4041   "TARGET_HAVE_MVE"
4042   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4043   [(set_attr "type" "mve_move")
4044 ])
4045
4046 ;;
4047 ;; [vmlsdavaxq_s])
4048 ;;
4049 (define_insn "mve_vmlsdavaxq_s<mode>"
4050   [
4051    (set (match_operand:SI 0 "s_register_operand" "=Te")
4052         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4053                     (match_operand:MVE_2 2 "s_register_operand" "w")
4054                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4055          VMLSDAVAXQ_S))
4056   ]
4057   "TARGET_HAVE_MVE"
4058   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4059   [(set_attr "type" "mve_move")
4060 ])
4061
4062 ;;
4063 ;; [vmlsdavaq_s])
4064 ;;
4065 (define_insn "mve_vmlsdavaq_s<mode>"
4066   [
4067    (set (match_operand:SI 0 "s_register_operand" "=Te")
4068         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4069                     (match_operand:MVE_2 2 "s_register_operand" "w")
4070                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4071          VMLSDAVAQ_S))
4072   ]
4073   "TARGET_HAVE_MVE"
4074   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4075   [(set_attr "type" "mve_move")
4076 ])
4077
4078 ;;
4079 ;; [vmladavaxq_s])
4080 ;;
4081 (define_insn "mve_vmladavaxq_s<mode>"
4082   [
4083    (set (match_operand:SI 0 "s_register_operand" "=Te")
4084         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4085                     (match_operand:MVE_2 2 "s_register_operand" "w")
4086                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4087          VMLADAVAXQ_S))
4088   ]
4089   "TARGET_HAVE_MVE"
4090   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4091   [(set_attr "type" "mve_move")
4092 ])
4093 ;;
4094 ;; [vabsq_m_f])
4095 ;;
4096 (define_insn "mve_vabsq_m_f<mode>"
4097   [
4098    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4099         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4100                        (match_operand:MVE_0 2 "s_register_operand" "w")
4101                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4102          VABSQ_M_F))
4103   ]
4104   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4105   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4106   [(set_attr "type" "mve_move")
4107    (set_attr "length""8")])
4108
4109 ;;
4110 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4111 ;;
4112 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4113   [
4114    (set (match_operand:DI 0 "s_register_operand" "=r")
4115         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4116                        (match_operand:V4SI 2 "s_register_operand" "w")
4117                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4118          VADDLVAQ_P))
4119   ]
4120   "TARGET_HAVE_MVE"
4121   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4122   [(set_attr "type" "mve_move")
4123    (set_attr "length""8")])
4124 ;;
4125 ;; [vcmlaq_f])
4126 ;;
4127 (define_insn "mve_vcmlaq_f<mode>"
4128   [
4129    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4130         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4131                        (match_operand:MVE_0 2 "s_register_operand" "w")
4132                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4133          VCMLAQ_F))
4134   ]
4135   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4136   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4137   [(set_attr "type" "mve_move")
4138 ])
4139
4140 ;;
4141 ;; [vcmlaq_rot180_f])
4142 ;;
4143 (define_insn "mve_vcmlaq_rot180_f<mode>"
4144   [
4145    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4146         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4147                        (match_operand:MVE_0 2 "s_register_operand" "w")
4148                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4149          VCMLAQ_ROT180_F))
4150   ]
4151   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4152   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4153   [(set_attr "type" "mve_move")
4154 ])
4155
4156 ;;
4157 ;; [vcmlaq_rot270_f])
4158 ;;
4159 (define_insn "mve_vcmlaq_rot270_f<mode>"
4160   [
4161    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4162         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4163                        (match_operand:MVE_0 2 "s_register_operand" "w")
4164                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4165          VCMLAQ_ROT270_F))
4166   ]
4167   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4168   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4169   [(set_attr "type" "mve_move")
4170 ])
4171
4172 ;;
4173 ;; [vcmlaq_rot90_f])
4174 ;;
4175 (define_insn "mve_vcmlaq_rot90_f<mode>"
4176   [
4177    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4178         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4179                        (match_operand:MVE_0 2 "s_register_operand" "w")
4180                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4181          VCMLAQ_ROT90_F))
4182   ]
4183   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4184   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4185   [(set_attr "type" "mve_move")
4186 ])
4187
4188 ;;
4189 ;; [vcmpeqq_m_n_f])
4190 ;;
4191 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4192   [
4193    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4194         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4195                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4196                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4197          VCMPEQQ_M_N_F))
4198   ]
4199   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4200   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4201   [(set_attr "type" "mve_move")
4202    (set_attr "length""8")])
4203
4204 ;;
4205 ;; [vcmpgeq_m_f])
4206 ;;
4207 (define_insn "mve_vcmpgeq_m_f<mode>"
4208   [
4209    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4210         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4211                        (match_operand:MVE_0 2 "s_register_operand" "w")
4212                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4213          VCMPGEQ_M_F))
4214   ]
4215   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4216   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4217   [(set_attr "type" "mve_move")
4218    (set_attr "length""8")])
4219
4220 ;;
4221 ;; [vcmpgeq_m_n_f])
4222 ;;
4223 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4224   [
4225    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4226         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4227                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4228                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4229          VCMPGEQ_M_N_F))
4230   ]
4231   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4232   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4233   [(set_attr "type" "mve_move")
4234    (set_attr "length""8")])
4235
4236 ;;
4237 ;; [vcmpgtq_m_f])
4238 ;;
4239 (define_insn "mve_vcmpgtq_m_f<mode>"
4240   [
4241    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4242         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4243                        (match_operand:MVE_0 2 "s_register_operand" "w")
4244                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4245          VCMPGTQ_M_F))
4246   ]
4247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4248   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4249   [(set_attr "type" "mve_move")
4250    (set_attr "length""8")])
4251
4252 ;;
4253 ;; [vcmpgtq_m_n_f])
4254 ;;
4255 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4256   [
4257    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4258         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4259                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4260                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4261          VCMPGTQ_M_N_F))
4262   ]
4263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4264   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4265   [(set_attr "type" "mve_move")
4266    (set_attr "length""8")])
4267
4268 ;;
4269 ;; [vcmpleq_m_f])
4270 ;;
4271 (define_insn "mve_vcmpleq_m_f<mode>"
4272   [
4273    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4274         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4275                        (match_operand:MVE_0 2 "s_register_operand" "w")
4276                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4277          VCMPLEQ_M_F))
4278   ]
4279   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4280   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4281   [(set_attr "type" "mve_move")
4282    (set_attr "length""8")])
4283
4284 ;;
4285 ;; [vcmpleq_m_n_f])
4286 ;;
4287 (define_insn "mve_vcmpleq_m_n_f<mode>"
4288   [
4289    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4290         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4291                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4292                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4293          VCMPLEQ_M_N_F))
4294   ]
4295   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4296   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4297   [(set_attr "type" "mve_move")
4298    (set_attr "length""8")])
4299
4300 ;;
4301 ;; [vcmpltq_m_f])
4302 ;;
4303 (define_insn "mve_vcmpltq_m_f<mode>"
4304   [
4305    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4306         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4307                        (match_operand:MVE_0 2 "s_register_operand" "w")
4308                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4309          VCMPLTQ_M_F))
4310   ]
4311   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4312   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4313   [(set_attr "type" "mve_move")
4314    (set_attr "length""8")])
4315
4316 ;;
4317 ;; [vcmpltq_m_n_f])
4318 ;;
4319 (define_insn "mve_vcmpltq_m_n_f<mode>"
4320   [
4321    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4322         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4323                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4324                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4325          VCMPLTQ_M_N_F))
4326   ]
4327   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4328   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4329   [(set_attr "type" "mve_move")
4330    (set_attr "length""8")])
4331
4332 ;;
4333 ;; [vcmpneq_m_f])
4334 ;;
4335 (define_insn "mve_vcmpneq_m_f<mode>"
4336   [
4337    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4338         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4339                        (match_operand:MVE_0 2 "s_register_operand" "w")
4340                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4341          VCMPNEQ_M_F))
4342   ]
4343   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4344   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4345   [(set_attr "type" "mve_move")
4346    (set_attr "length""8")])
4347
4348 ;;
4349 ;; [vcmpneq_m_n_f])
4350 ;;
4351 (define_insn "mve_vcmpneq_m_n_f<mode>"
4352   [
4353    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4354         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4355                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4356                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4357          VCMPNEQ_M_N_F))
4358   ]
4359   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4360   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4361   [(set_attr "type" "mve_move")
4362    (set_attr "length""8")])
4363
4364 ;;
4365 ;; [vcvtbq_m_f16_f32])
4366 ;;
4367 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4368   [
4369    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4370         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4371                        (match_operand:V4SF 2 "s_register_operand" "w")
4372                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4373          VCVTBQ_M_F16_F32))
4374   ]
4375   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4376   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4377   [(set_attr "type" "mve_move")
4378    (set_attr "length""8")])
4379
4380 ;;
4381 ;; [vcvtbq_m_f32_f16])
4382 ;;
4383 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4384   [
4385    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4386         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4387                        (match_operand:V8HF 2 "s_register_operand" "w")
4388                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4389          VCVTBQ_M_F32_F16))
4390   ]
4391   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4392   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4393   [(set_attr "type" "mve_move")
4394    (set_attr "length""8")])
4395
4396 ;;
4397 ;; [vcvttq_m_f16_f32])
4398 ;;
4399 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4400   [
4401    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4402         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4403                        (match_operand:V4SF 2 "s_register_operand" "w")
4404                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4405          VCVTTQ_M_F16_F32))
4406   ]
4407   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4408   "vpst\;vcvttt.f16.f32 %q0, %q2"
4409   [(set_attr "type" "mve_move")
4410    (set_attr "length""8")])
4411
4412 ;;
4413 ;; [vcvttq_m_f32_f16])
4414 ;;
4415 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4416   [
4417    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4418         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4419                        (match_operand:V8HF 2 "s_register_operand" "w")
4420                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4421          VCVTTQ_M_F32_F16))
4422   ]
4423   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4424   "vpst\;vcvttt.f32.f16 %q0, %q2"
4425   [(set_attr "type" "mve_move")
4426    (set_attr "length""8")])
4427
4428 ;;
4429 ;; [vdupq_m_n_f])
4430 ;;
4431 (define_insn "mve_vdupq_m_n_f<mode>"
4432   [
4433    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4434         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4435                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4436                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4437          VDUPQ_M_N_F))
4438   ]
4439   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4440   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4441   [(set_attr "type" "mve_move")
4442    (set_attr "length""8")])
4443
4444 ;;
4445 ;; [vfmaq_f])
4446 ;;
4447 (define_insn "mve_vfmaq_f<mode>"
4448   [
4449    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4450         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4451                        (match_operand:MVE_0 2 "s_register_operand" "w")
4452                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4453          VFMAQ_F))
4454   ]
4455   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4456   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4457   [(set_attr "type" "mve_move")
4458 ])
4459
4460 ;;
4461 ;; [vfmaq_n_f])
4462 ;;
4463 (define_insn "mve_vfmaq_n_f<mode>"
4464   [
4465    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4466         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4467                        (match_operand:MVE_0 2 "s_register_operand" "w")
4468                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4469          VFMAQ_N_F))
4470   ]
4471   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4472   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4473   [(set_attr "type" "mve_move")
4474 ])
4475
4476 ;;
4477 ;; [vfmasq_n_f])
4478 ;;
4479 (define_insn "mve_vfmasq_n_f<mode>"
4480   [
4481    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4482         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4483                        (match_operand:MVE_0 2 "s_register_operand" "w")
4484                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4485          VFMASQ_N_F))
4486   ]
4487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4488   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4489   [(set_attr "type" "mve_move")
4490 ])
4491 ;;
4492 ;; [vfmsq_f])
4493 ;;
4494 (define_insn "mve_vfmsq_f<mode>"
4495   [
4496    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4497         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4498                        (match_operand:MVE_0 2 "s_register_operand" "w")
4499                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4500          VFMSQ_F))
4501   ]
4502   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4503   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4504   [(set_attr "type" "mve_move")
4505 ])
4506
4507 ;;
4508 ;; [vmaxnmaq_m_f])
4509 ;;
4510 (define_insn "mve_vmaxnmaq_m_f<mode>"
4511   [
4512    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4513         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4514                        (match_operand:MVE_0 2 "s_register_operand" "w")
4515                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4516          VMAXNMAQ_M_F))
4517   ]
4518   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4519   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4520   [(set_attr "type" "mve_move")
4521    (set_attr "length""8")])
4522 ;;
4523 ;; [vmaxnmavq_p_f])
4524 ;;
4525 (define_insn "mve_vmaxnmavq_p_f<mode>"
4526   [
4527    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4528         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4529                        (match_operand:MVE_0 2 "s_register_operand" "w")
4530                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4531          VMAXNMAVQ_P_F))
4532   ]
4533   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4534   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4535   [(set_attr "type" "mve_move")
4536    (set_attr "length""8")])
4537
4538 ;;
4539 ;; [vmaxnmvq_p_f])
4540 ;;
4541 (define_insn "mve_vmaxnmvq_p_f<mode>"
4542   [
4543    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4544         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4545                        (match_operand:MVE_0 2 "s_register_operand" "w")
4546                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4547          VMAXNMVQ_P_F))
4548   ]
4549   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4550   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4551   [(set_attr "type" "mve_move")
4552    (set_attr "length""8")])
4553 ;;
4554 ;; [vminnmaq_m_f])
4555 ;;
4556 (define_insn "mve_vminnmaq_m_f<mode>"
4557   [
4558    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4559         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4560                        (match_operand:MVE_0 2 "s_register_operand" "w")
4561                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4562          VMINNMAQ_M_F))
4563   ]
4564   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4565   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4566   [(set_attr "type" "mve_move")
4567    (set_attr "length""8")])
4568
4569 ;;
4570 ;; [vminnmavq_p_f])
4571 ;;
4572 (define_insn "mve_vminnmavq_p_f<mode>"
4573   [
4574    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4575         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4576                        (match_operand:MVE_0 2 "s_register_operand" "w")
4577                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4578          VMINNMAVQ_P_F))
4579   ]
4580   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4581   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4582   [(set_attr "type" "mve_move")
4583    (set_attr "length""8")])
4584 ;;
4585 ;; [vminnmvq_p_f])
4586 ;;
4587 (define_insn "mve_vminnmvq_p_f<mode>"
4588   [
4589    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4590         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4591                        (match_operand:MVE_0 2 "s_register_operand" "w")
4592                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4593          VMINNMVQ_P_F))
4594   ]
4595   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4596   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4597   [(set_attr "type" "mve_move")
4598    (set_attr "length""8")])
4599
4600 ;;
4601 ;; [vmlaldavaq_s, vmlaldavaq_u])
4602 ;;
4603 (define_insn "mve_vmlaldavaq_<supf><mode>"
4604   [
4605    (set (match_operand:DI 0 "s_register_operand" "=r")
4606         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4607                        (match_operand:MVE_5 2 "s_register_operand" "w")
4608                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4609          VMLALDAVAQ))
4610   ]
4611   "TARGET_HAVE_MVE"
4612   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4613   [(set_attr "type" "mve_move")
4614 ])
4615
4616 ;;
4617 ;; [vmlaldavaxq_s])
4618 ;;
4619 (define_insn "mve_vmlaldavaxq_s<mode>"
4620   [
4621    (set (match_operand:DI 0 "s_register_operand" "=r")
4622         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4623                        (match_operand:MVE_5 2 "s_register_operand" "w")
4624                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4625          VMLALDAVAXQ_S))
4626   ]
4627   "TARGET_HAVE_MVE"
4628   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4629   [(set_attr "type" "mve_move")
4630 ])
4631
4632 ;;
4633 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4634 ;;
4635 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4636   [
4637    (set (match_operand:DI 0 "s_register_operand" "=r")
4638         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4639                        (match_operand:MVE_5 2 "s_register_operand" "w")
4640                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4641          VMLALDAVQ_P))
4642   ]
4643   "TARGET_HAVE_MVE"
4644   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4645   [(set_attr "type" "mve_move")
4646    (set_attr "length""8")])
4647
4648 ;;
4649 ;; [vmlaldavxq_p_s])
4650 ;;
4651 (define_insn "mve_vmlaldavxq_p_s<mode>"
4652   [
4653    (set (match_operand:DI 0 "s_register_operand" "=r")
4654         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4655                        (match_operand:MVE_5 2 "s_register_operand" "w")
4656                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4657          VMLALDAVXQ_P_S))
4658   ]
4659   "TARGET_HAVE_MVE"
4660   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4661   [(set_attr "type" "mve_move")
4662    (set_attr "length""8")])
4663 ;;
4664 ;; [vmlsldavaq_s])
4665 ;;
4666 (define_insn "mve_vmlsldavaq_s<mode>"
4667   [
4668    (set (match_operand:DI 0 "s_register_operand" "=r")
4669         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4670                        (match_operand:MVE_5 2 "s_register_operand" "w")
4671                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4672          VMLSLDAVAQ_S))
4673   ]
4674   "TARGET_HAVE_MVE"
4675   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4676   [(set_attr "type" "mve_move")
4677 ])
4678
4679 ;;
4680 ;; [vmlsldavaxq_s])
4681 ;;
4682 (define_insn "mve_vmlsldavaxq_s<mode>"
4683   [
4684    (set (match_operand:DI 0 "s_register_operand" "=r")
4685         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4686                        (match_operand:MVE_5 2 "s_register_operand" "w")
4687                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4688          VMLSLDAVAXQ_S))
4689   ]
4690   "TARGET_HAVE_MVE"
4691   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4692   [(set_attr "type" "mve_move")
4693 ])
4694
4695 ;;
4696 ;; [vmlsldavq_p_s])
4697 ;;
4698 (define_insn "mve_vmlsldavq_p_s<mode>"
4699   [
4700    (set (match_operand:DI 0 "s_register_operand" "=r")
4701         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4702                        (match_operand:MVE_5 2 "s_register_operand" "w")
4703                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4704          VMLSLDAVQ_P_S))
4705   ]
4706   "TARGET_HAVE_MVE"
4707   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4708   [(set_attr "type" "mve_move")
4709    (set_attr "length""8")])
4710
4711 ;;
4712 ;; [vmlsldavxq_p_s])
4713 ;;
4714 (define_insn "mve_vmlsldavxq_p_s<mode>"
4715   [
4716    (set (match_operand:DI 0 "s_register_operand" "=r")
4717         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4718                        (match_operand:MVE_5 2 "s_register_operand" "w")
4719                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4720          VMLSLDAVXQ_P_S))
4721   ]
4722   "TARGET_HAVE_MVE"
4723   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4724   [(set_attr "type" "mve_move")
4725    (set_attr "length""8")])
4726 ;;
4727 ;; [vmovlbq_m_u, vmovlbq_m_s])
4728 ;;
4729 (define_insn "mve_vmovlbq_m_<supf><mode>"
4730   [
4731    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4732         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4733                        (match_operand:MVE_3 2 "s_register_operand" "w")
4734                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4735          VMOVLBQ_M))
4736   ]
4737   "TARGET_HAVE_MVE"
4738   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4739   [(set_attr "type" "mve_move")
4740    (set_attr "length""8")])
4741 ;;
4742 ;; [vmovltq_m_u, vmovltq_m_s])
4743 ;;
4744 (define_insn "mve_vmovltq_m_<supf><mode>"
4745   [
4746    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4747         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4748                        (match_operand:MVE_3 2 "s_register_operand" "w")
4749                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4750          VMOVLTQ_M))
4751   ]
4752   "TARGET_HAVE_MVE"
4753   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4754   [(set_attr "type" "mve_move")
4755    (set_attr "length""8")])
4756 ;;
4757 ;; [vmovnbq_m_u, vmovnbq_m_s])
4758 ;;
4759 (define_insn "mve_vmovnbq_m_<supf><mode>"
4760   [
4761    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4762         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4763                        (match_operand:MVE_5 2 "s_register_operand" "w")
4764                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4765          VMOVNBQ_M))
4766   ]
4767   "TARGET_HAVE_MVE"
4768   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4769   [(set_attr "type" "mve_move")
4770    (set_attr "length""8")])
4771
4772 ;;
4773 ;; [vmovntq_m_u, vmovntq_m_s])
4774 ;;
4775 (define_insn "mve_vmovntq_m_<supf><mode>"
4776   [
4777    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4778         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4779                        (match_operand:MVE_5 2 "s_register_operand" "w")
4780                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4781          VMOVNTQ_M))
4782   ]
4783   "TARGET_HAVE_MVE"
4784   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4785   [(set_attr "type" "mve_move")
4786    (set_attr "length""8")])
4787
4788 ;;
4789 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4790 ;;
4791 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4792   [
4793    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4794         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4795                        (match_operand:SI 2 "immediate_operand" "i")
4796                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4797          VMVNQ_M_N))
4798   ]
4799   "TARGET_HAVE_MVE"
4800   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4801   [(set_attr "type" "mve_move")
4802    (set_attr "length""8")])
4803 ;;
4804 ;; [vnegq_m_f])
4805 ;;
4806 (define_insn "mve_vnegq_m_f<mode>"
4807   [
4808    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4809         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4810                        (match_operand:MVE_0 2 "s_register_operand" "w")
4811                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4812          VNEGQ_M_F))
4813   ]
4814   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4815   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4816   [(set_attr "type" "mve_move")
4817    (set_attr "length""8")])
4818
4819 ;;
4820 ;; [vorrq_m_n_s, vorrq_m_n_u])
4821 ;;
4822 (define_insn "mve_vorrq_m_n_<supf><mode>"
4823   [
4824    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4825         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4826                        (match_operand:SI 2 "immediate_operand" "i")
4827                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4828          VORRQ_M_N))
4829   ]
4830   "TARGET_HAVE_MVE"
4831   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4832   [(set_attr "type" "mve_move")
4833    (set_attr "length""8")])
4834 ;;
4835 ;; [vpselq_f])
4836 ;;
4837 (define_insn "mve_vpselq_f<mode>"
4838   [
4839    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4840         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4841                        (match_operand:MVE_0 2 "s_register_operand" "w")
4842                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4843          VPSELQ_F))
4844   ]
4845   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4846   "vpsel %q0, %q1, %q2"
4847   [(set_attr "type" "mve_move")
4848 ])
4849
4850 ;;
4851 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4852 ;;
4853 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4854   [
4855    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4856         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4857                        (match_operand:MVE_5 2 "s_register_operand" "w")
4858                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4859          VQMOVNBQ_M))
4860   ]
4861   "TARGET_HAVE_MVE"
4862   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4863   [(set_attr "type" "mve_move")
4864    (set_attr "length""8")])
4865
4866 ;;
4867 ;; [vqmovntq_m_u, vqmovntq_m_s])
4868 ;;
4869 (define_insn "mve_vqmovntq_m_<supf><mode>"
4870   [
4871    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4872         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4873                        (match_operand:MVE_5 2 "s_register_operand" "w")
4874                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4875          VQMOVNTQ_M))
4876   ]
4877   "TARGET_HAVE_MVE"
4878   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4879   [(set_attr "type" "mve_move")
4880    (set_attr "length""8")])
4881
4882 ;;
4883 ;; [vqmovunbq_m_s])
4884 ;;
4885 (define_insn "mve_vqmovunbq_m_s<mode>"
4886   [
4887    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4888         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4889                        (match_operand:MVE_5 2 "s_register_operand" "w")
4890                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4891          VQMOVUNBQ_M_S))
4892   ]
4893   "TARGET_HAVE_MVE"
4894   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4895   [(set_attr "type" "mve_move")
4896    (set_attr "length""8")])
4897
4898 ;;
4899 ;; [vqmovuntq_m_s])
4900 ;;
4901 (define_insn "mve_vqmovuntq_m_s<mode>"
4902   [
4903    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4904         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4905                        (match_operand:MVE_5 2 "s_register_operand" "w")
4906                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4907          VQMOVUNTQ_M_S))
4908   ]
4909   "TARGET_HAVE_MVE"
4910   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4911   [(set_attr "type" "mve_move")
4912    (set_attr "length""8")])
4913
4914 ;;
4915 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4916 ;;
4917 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4918   [
4919    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4920         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4921                        (match_operand:MVE_5 2 "s_register_operand" "w")
4922                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4923          VQRSHRNTQ_N))
4924   ]
4925   "TARGET_HAVE_MVE"
4926   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4927   [(set_attr "type" "mve_move")
4928 ])
4929
4930 ;;
4931 ;; [vqrshruntq_n_s])
4932 ;;
4933 (define_insn "mve_vqrshruntq_n_s<mode>"
4934   [
4935    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4936         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4937                        (match_operand:MVE_5 2 "s_register_operand" "w")
4938                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4939          VQRSHRUNTQ_N_S))
4940   ]
4941   "TARGET_HAVE_MVE"
4942   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4943   [(set_attr "type" "mve_move")
4944 ])
4945
4946 ;;
4947 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4948 ;;
4949 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4950   [
4951    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4952         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4953                        (match_operand:MVE_5 2 "s_register_operand" "w")
4954                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4955          VQSHRNBQ_N))
4956   ]
4957   "TARGET_HAVE_MVE"
4958   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4959   [(set_attr "type" "mve_move")
4960 ])
4961
4962 ;;
4963 ;; [vqshrntq_n_u, vqshrntq_n_s])
4964 ;;
4965 (define_insn "mve_vqshrntq_n_<supf><mode>"
4966   [
4967    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4968         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4969                        (match_operand:MVE_5 2 "s_register_operand" "w")
4970                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4971          VQSHRNTQ_N))
4972   ]
4973   "TARGET_HAVE_MVE"
4974   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4975   [(set_attr "type" "mve_move")
4976 ])
4977
4978 ;;
4979 ;; [vqshrunbq_n_s])
4980 ;;
4981 (define_insn "mve_vqshrunbq_n_s<mode>"
4982   [
4983    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4984         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4985                        (match_operand:MVE_5 2 "s_register_operand" "w")
4986                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4987          VQSHRUNBQ_N_S))
4988   ]
4989   "TARGET_HAVE_MVE"
4990   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4991   [(set_attr "type" "mve_move")
4992 ])
4993
4994 ;;
4995 ;; [vqshruntq_n_s])
4996 ;;
4997 (define_insn "mve_vqshruntq_n_s<mode>"
4998   [
4999    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5000         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5001                        (match_operand:MVE_5 2 "s_register_operand" "w")
5002                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5003          VQSHRUNTQ_N_S))
5004   ]
5005   "TARGET_HAVE_MVE"
5006   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5007   [(set_attr "type" "mve_move")
5008 ])
5009
5010 ;;
5011 ;; [vrev32q_m_f])
5012 ;;
5013 (define_insn "mve_vrev32q_m_fv8hf"
5014   [
5015    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5016         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5017                        (match_operand:V8HF 2 "s_register_operand" "w")
5018                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5019          VREV32Q_M_F))
5020   ]
5021   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5022   "vpst\;vrev32t.16 %q0, %q2"
5023   [(set_attr "type" "mve_move")
5024    (set_attr "length""8")])
5025
5026 ;;
5027 ;; [vrev32q_m_s, vrev32q_m_u])
5028 ;;
5029 (define_insn "mve_vrev32q_m_<supf><mode>"
5030   [
5031    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5032         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5033                        (match_operand:MVE_3 2 "s_register_operand" "w")
5034                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5035          VREV32Q_M))
5036   ]
5037   "TARGET_HAVE_MVE"
5038   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5039   [(set_attr "type" "mve_move")
5040    (set_attr "length""8")])
5041
5042 ;;
5043 ;; [vrev64q_m_f])
5044 ;;
5045 (define_insn "mve_vrev64q_m_f<mode>"
5046   [
5047    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5048         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5049                        (match_operand:MVE_0 2 "s_register_operand" "w")
5050                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5051          VREV64Q_M_F))
5052   ]
5053   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5054   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5055   [(set_attr "type" "mve_move")
5056    (set_attr "length""8")])
5057
5058 ;;
5059 ;; [vrmlaldavhaxq_s])
5060 ;;
5061 (define_insn "mve_vrmlaldavhaxq_sv4si"
5062   [
5063    (set (match_operand:DI 0 "s_register_operand" "=r")
5064         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5065                        (match_operand:V4SI 2 "s_register_operand" "w")
5066                        (match_operand:V4SI 3 "s_register_operand" "w")]
5067          VRMLALDAVHAXQ_S))
5068   ]
5069   "TARGET_HAVE_MVE"
5070   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5071   [(set_attr "type" "mve_move")
5072 ])
5073
5074 ;;
5075 ;; [vrmlaldavhxq_p_s])
5076 ;;
5077 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5078   [
5079    (set (match_operand:DI 0 "s_register_operand" "=r")
5080         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5081                        (match_operand:V4SI 2 "s_register_operand" "w")
5082                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5083          VRMLALDAVHXQ_P_S))
5084   ]
5085   "TARGET_HAVE_MVE"
5086   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5087   [(set_attr "type" "mve_move")
5088    (set_attr "length""8")])
5089
5090 ;;
5091 ;; [vrmlsldavhaxq_s])
5092 ;;
5093 (define_insn "mve_vrmlsldavhaxq_sv4si"
5094   [
5095    (set (match_operand:DI 0 "s_register_operand" "=r")
5096         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5097                        (match_operand:V4SI 2 "s_register_operand" "w")
5098                        (match_operand:V4SI 3 "s_register_operand" "w")]
5099          VRMLSLDAVHAXQ_S))
5100   ]
5101   "TARGET_HAVE_MVE"
5102   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5103   [(set_attr "type" "mve_move")
5104 ])
5105
5106 ;;
5107 ;; [vrmlsldavhq_p_s])
5108 ;;
5109 (define_insn "mve_vrmlsldavhq_p_sv4si"
5110   [
5111    (set (match_operand:DI 0 "s_register_operand" "=r")
5112         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5113                        (match_operand:V4SI 2 "s_register_operand" "w")
5114                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5115          VRMLSLDAVHQ_P_S))
5116   ]
5117   "TARGET_HAVE_MVE"
5118   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5119   [(set_attr "type" "mve_move")
5120    (set_attr "length""8")])
5121
5122 ;;
5123 ;; [vrmlsldavhxq_p_s])
5124 ;;
5125 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5126   [
5127    (set (match_operand:DI 0 "s_register_operand" "=r")
5128         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5129                        (match_operand:V4SI 2 "s_register_operand" "w")
5130                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5131          VRMLSLDAVHXQ_P_S))
5132   ]
5133   "TARGET_HAVE_MVE"
5134   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5135   [(set_attr "type" "mve_move")
5136    (set_attr "length""8")])
5137
5138 ;;
5139 ;; [vrndaq_m_f])
5140 ;;
5141 (define_insn "mve_vrndaq_m_f<mode>"
5142   [
5143    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5144         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5145                        (match_operand:MVE_0 2 "s_register_operand" "w")
5146                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5147          VRNDAQ_M_F))
5148   ]
5149   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5150   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5151   [(set_attr "type" "mve_move")
5152    (set_attr "length""8")])
5153
5154 ;;
5155 ;; [vrndmq_m_f])
5156 ;;
5157 (define_insn "mve_vrndmq_m_f<mode>"
5158   [
5159    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5160         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5161                        (match_operand:MVE_0 2 "s_register_operand" "w")
5162                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5163          VRNDMQ_M_F))
5164   ]
5165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5166   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5167   [(set_attr "type" "mve_move")
5168    (set_attr "length""8")])
5169
5170 ;;
5171 ;; [vrndnq_m_f])
5172 ;;
5173 (define_insn "mve_vrndnq_m_f<mode>"
5174   [
5175    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5176         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5177                        (match_operand:MVE_0 2 "s_register_operand" "w")
5178                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5179          VRNDNQ_M_F))
5180   ]
5181   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5182   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5183   [(set_attr "type" "mve_move")
5184    (set_attr "length""8")])
5185
5186 ;;
5187 ;; [vrndpq_m_f])
5188 ;;
5189 (define_insn "mve_vrndpq_m_f<mode>"
5190   [
5191    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5192         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5193                        (match_operand:MVE_0 2 "s_register_operand" "w")
5194                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5195          VRNDPQ_M_F))
5196   ]
5197   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5198   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5199   [(set_attr "type" "mve_move")
5200    (set_attr "length""8")])
5201
5202 ;;
5203 ;; [vrndxq_m_f])
5204 ;;
5205 (define_insn "mve_vrndxq_m_f<mode>"
5206   [
5207    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5208         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5209                        (match_operand:MVE_0 2 "s_register_operand" "w")
5210                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5211          VRNDXQ_M_F))
5212   ]
5213   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5214   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5215   [(set_attr "type" "mve_move")
5216    (set_attr "length""8")])
5217
5218 ;;
5219 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5220 ;;
5221 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5222   [
5223    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5224         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5225                        (match_operand:MVE_5 2 "s_register_operand" "w")
5226                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5227          VRSHRNBQ_N))
5228   ]
5229   "TARGET_HAVE_MVE"
5230   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5231   [(set_attr "type" "mve_move")
5232 ])
5233
5234 ;;
5235 ;; [vrshrntq_n_u, vrshrntq_n_s])
5236 ;;
5237 (define_insn "mve_vrshrntq_n_<supf><mode>"
5238   [
5239    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5240         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5241                        (match_operand:MVE_5 2 "s_register_operand" "w")
5242                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5243          VRSHRNTQ_N))
5244   ]
5245   "TARGET_HAVE_MVE"
5246   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5247   [(set_attr "type" "mve_move")
5248 ])
5249
5250 ;;
5251 ;; [vshrnbq_n_u, vshrnbq_n_s])
5252 ;;
5253 (define_insn "mve_vshrnbq_n_<supf><mode>"
5254   [
5255    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5256         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5257                        (match_operand:MVE_5 2 "s_register_operand" "w")
5258                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5259          VSHRNBQ_N))
5260   ]
5261   "TARGET_HAVE_MVE"
5262   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5263   [(set_attr "type" "mve_move")
5264 ])
5265
5266 ;;
5267 ;; [vshrntq_n_s, vshrntq_n_u])
5268 ;;
5269 (define_insn "mve_vshrntq_n_<supf><mode>"
5270   [
5271    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5272         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5273                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5274                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5275          VSHRNTQ_N))
5276   ]
5277   "TARGET_HAVE_MVE"
5278   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5279   [(set_attr "type" "mve_move")
5280 ])
5281
5282 ;;
5283 ;; [vcvtmq_m_s, vcvtmq_m_u])
5284 ;;
5285 (define_insn "mve_vcvtmq_m_<supf><mode>"
5286   [
5287    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5288         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5289                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5290                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5291          VCVTMQ_M))
5292   ]
5293   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5294   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5295   [(set_attr "type" "mve_move")
5296    (set_attr "length""8")])
5297
5298 ;;
5299 ;; [vcvtpq_m_u, vcvtpq_m_s])
5300 ;;
5301 (define_insn "mve_vcvtpq_m_<supf><mode>"
5302   [
5303    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5304         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5305                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5306                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5307          VCVTPQ_M))
5308   ]
5309   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5310   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5311   [(set_attr "type" "mve_move")
5312    (set_attr "length""8")])
5313
5314 ;;
5315 ;; [vcvtnq_m_s, vcvtnq_m_u])
5316 ;;
5317 (define_insn "mve_vcvtnq_m_<supf><mode>"
5318   [
5319    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5320         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5321                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5322                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5323          VCVTNQ_M))
5324   ]
5325   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5326   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5327   [(set_attr "type" "mve_move")
5328    (set_attr "length""8")])
5329
5330 ;;
5331 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5332 ;;
5333 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5334   [
5335    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5336         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5337                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5338                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5339                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5340          VCVTQ_M_N_FROM_F))
5341   ]
5342   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5343   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5344   [(set_attr "type" "mve_move")
5345    (set_attr "length""8")])
5346
5347 ;;
5348 ;; [vrev16q_m_u, vrev16q_m_s])
5349 ;;
5350 (define_insn "mve_vrev16q_m_<supf>v16qi"
5351   [
5352    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5353         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5354                        (match_operand:V16QI 2 "s_register_operand" "w")
5355                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5356          VREV16Q_M))
5357   ]
5358   "TARGET_HAVE_MVE"
5359   "vpst\;vrev16t.8 %q0, %q2"
5360   [(set_attr "type" "mve_move")
5361    (set_attr "length""8")])
5362
5363 ;;
5364 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5365 ;;
5366 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5367   [
5368    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5369         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5370                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5371                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5372          VCVTQ_M_FROM_F))
5373   ]
5374   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5375   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5376   [(set_attr "type" "mve_move")
5377    (set_attr "length""8")])
5378
5379 ;;
5380 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5381 ;;
5382 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5383   [
5384    (set (match_operand:DI 0 "s_register_operand" "=r")
5385         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5386                     (match_operand:V4SI 2 "s_register_operand" "w")
5387                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5388          VRMLALDAVHQ_P))
5389   ]
5390   "TARGET_HAVE_MVE"
5391   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5392   [(set_attr "type" "mve_move")
5393    (set_attr "length""8")])
5394
5395 ;;
5396 ;; [vrmlsldavhaq_s])
5397 ;;
5398 (define_insn "mve_vrmlsldavhaq_sv4si"
5399   [
5400    (set (match_operand:DI 0 "s_register_operand" "=r")
5401         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5402                     (match_operand:V4SI 2 "s_register_operand" "w")
5403                     (match_operand:V4SI 3 "s_register_operand" "w")]
5404          VRMLSLDAVHAQ_S))
5405   ]
5406   "TARGET_HAVE_MVE"
5407   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5408   [(set_attr "type" "mve_move")
5409 ])
5410
5411 ;;
5412 ;; [vabavq_p_s, vabavq_p_u])
5413 ;;
5414 (define_insn "mve_vabavq_p_<supf><mode>"
5415   [
5416    (set (match_operand:SI 0 "s_register_operand" "=r")
5417         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5418                     (match_operand:MVE_2 2 "s_register_operand" "w")
5419                     (match_operand:MVE_2 3 "s_register_operand" "w")
5420                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5421          VABAVQ_P))
5422   ]
5423   "TARGET_HAVE_MVE"
5424   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5425   [(set_attr "type" "mve_move")
5426 ])
5427
5428 ;;
5429 ;; [vqshluq_m_n_s])
5430 ;;
5431 (define_insn "mve_vqshluq_m_n_s<mode>"
5432   [
5433    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5434         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5435                        (match_operand:MVE_2 2 "s_register_operand" "w")
5436                        (match_operand:SI 3 "mve_imm_7" "Ra")
5437                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5438          VQSHLUQ_M_N_S))
5439   ]
5440   "TARGET_HAVE_MVE"
5441   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5442   [(set_attr "type" "mve_move")])
5443
5444 ;;
5445 ;; [vshlq_m_s, vshlq_m_u])
5446 ;;
5447 (define_insn "mve_vshlq_m_<supf><mode>"
5448   [
5449    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5450         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5451                        (match_operand:MVE_2 2 "s_register_operand" "w")
5452                        (match_operand:MVE_2 3 "s_register_operand" "w")
5453                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5454          VSHLQ_M))
5455   ]
5456   "TARGET_HAVE_MVE"
5457   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5458   [(set_attr "type" "mve_move")])
5459
5460 ;;
5461 ;; [vsriq_m_n_s, vsriq_m_n_u])
5462 ;;
5463 (define_insn "mve_vsriq_m_n_<supf><mode>"
5464   [
5465    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5466         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5467                        (match_operand:MVE_2 2 "s_register_operand" "w")
5468                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5469                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5470          VSRIQ_M_N))
5471   ]
5472   "TARGET_HAVE_MVE"
5473   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5474   [(set_attr "type" "mve_move")])
5475
5476 ;;
5477 ;; [vsubq_m_u, vsubq_m_s])
5478 ;;
5479 (define_insn "mve_vsubq_m_<supf><mode>"
5480   [
5481    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5482         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5483                        (match_operand:MVE_2 2 "s_register_operand" "w")
5484                        (match_operand:MVE_2 3 "s_register_operand" "w")
5485                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5486          VSUBQ_M))
5487   ]
5488   "TARGET_HAVE_MVE"
5489   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5490   [(set_attr "type" "mve_move")])
5491
5492 ;;
5493 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5494 ;;
5495 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5496   [
5497    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5498         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5499                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5500                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5501                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5502          VCVTQ_M_N_TO_F))
5503   ]
5504   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5505   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5506   [(set_attr "type" "mve_move")
5507    (set_attr "length""8")])
5508 ;;
5509 ;; [vabdq_m_s, vabdq_m_u])
5510 ;;
5511 (define_insn "mve_vabdq_m_<supf><mode>"
5512   [
5513    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5514         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5515                        (match_operand:MVE_2 2 "s_register_operand" "w")
5516                        (match_operand:MVE_2 3 "s_register_operand" "w")
5517                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5518          VABDQ_M))
5519   ]
5520   "TARGET_HAVE_MVE"
5521   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5522   [(set_attr "type" "mve_move")
5523    (set_attr "length""8")])
5524
5525 ;;
5526 ;; [vaddq_m_n_s, vaddq_m_n_u])
5527 ;;
5528 (define_insn "mve_vaddq_m_n_<supf><mode>"
5529   [
5530    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5531         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5532                        (match_operand:MVE_2 2 "s_register_operand" "w")
5533                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5534                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5535          VADDQ_M_N))
5536   ]
5537   "TARGET_HAVE_MVE"
5538   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5539   [(set_attr "type" "mve_move")
5540    (set_attr "length""8")])
5541
5542 ;;
5543 ;; [vaddq_m_u, vaddq_m_s])
5544 ;;
5545 (define_insn "mve_vaddq_m_<supf><mode>"
5546   [
5547    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5548         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5549                        (match_operand:MVE_2 2 "s_register_operand" "w")
5550                        (match_operand:MVE_2 3 "s_register_operand" "w")
5551                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5552          VADDQ_M))
5553   ]
5554   "TARGET_HAVE_MVE"
5555   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5556   [(set_attr "type" "mve_move")
5557    (set_attr "length""8")])
5558
5559 ;;
5560 ;; [vandq_m_u, vandq_m_s])
5561 ;;
5562 (define_insn "mve_vandq_m_<supf><mode>"
5563   [
5564    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5565         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5566                        (match_operand:MVE_2 2 "s_register_operand" "w")
5567                        (match_operand:MVE_2 3 "s_register_operand" "w")
5568                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5569          VANDQ_M))
5570   ]
5571   "TARGET_HAVE_MVE"
5572   "vpst\;vandt %q0, %q2, %q3"
5573   [(set_attr "type" "mve_move")
5574    (set_attr "length""8")])
5575
5576 ;;
5577 ;; [vbicq_m_u, vbicq_m_s])
5578 ;;
5579 (define_insn "mve_vbicq_m_<supf><mode>"
5580   [
5581    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5582         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5583                        (match_operand:MVE_2 2 "s_register_operand" "w")
5584                        (match_operand:MVE_2 3 "s_register_operand" "w")
5585                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5586          VBICQ_M))
5587   ]
5588   "TARGET_HAVE_MVE"
5589   "vpst\;vbict %q0, %q2, %q3"
5590   [(set_attr "type" "mve_move")
5591    (set_attr "length""8")])
5592
5593 ;;
5594 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5595 ;;
5596 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5597   [
5598    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5599         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5600                        (match_operand:MVE_2 2 "s_register_operand" "w")
5601                        (match_operand:SI 3 "s_register_operand" "r")
5602                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5603          VBRSRQ_M_N))
5604   ]
5605   "TARGET_HAVE_MVE"
5606   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5607   [(set_attr "type" "mve_move")
5608    (set_attr "length""8")])
5609
5610 ;;
5611 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5612 ;;
5613 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5614   [
5615    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5616         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5617                        (match_operand:MVE_2 2 "s_register_operand" "w")
5618                        (match_operand:MVE_2 3 "s_register_operand" "w")
5619                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5620          VCADDQ_ROT270_M))
5621   ]
5622   "TARGET_HAVE_MVE"
5623   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5624   [(set_attr "type" "mve_move")
5625    (set_attr "length""8")])
5626
5627 ;;
5628 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5629 ;;
5630 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5631   [
5632    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5633         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5634                        (match_operand:MVE_2 2 "s_register_operand" "w")
5635                        (match_operand:MVE_2 3 "s_register_operand" "w")
5636                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5637          VCADDQ_ROT90_M))
5638   ]
5639   "TARGET_HAVE_MVE"
5640   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5641   [(set_attr "type" "mve_move")
5642    (set_attr "length""8")])
5643
5644 ;;
5645 ;; [veorq_m_s, veorq_m_u])
5646 ;;
5647 (define_insn "mve_veorq_m_<supf><mode>"
5648   [
5649    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5650         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5651                        (match_operand:MVE_2 2 "s_register_operand" "w")
5652                        (match_operand:MVE_2 3 "s_register_operand" "w")
5653                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5654          VEORQ_M))
5655   ]
5656   "TARGET_HAVE_MVE"
5657   "vpst\;veort %q0, %q2, %q3"
5658   [(set_attr "type" "mve_move")
5659    (set_attr "length""8")])
5660
5661 ;;
5662 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5663 ;;
5664 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5665   [
5666    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5667         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5668                        (match_operand:MVE_2 2 "s_register_operand" "w")
5669                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5670                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5671          VHADDQ_M_N))
5672   ]
5673   "TARGET_HAVE_MVE"
5674   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5675   [(set_attr "type" "mve_move")
5676    (set_attr "length""8")])
5677
5678 ;;
5679 ;; [vhaddq_m_s, vhaddq_m_u])
5680 ;;
5681 (define_insn "mve_vhaddq_m_<supf><mode>"
5682   [
5683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5685                        (match_operand:MVE_2 2 "s_register_operand" "w")
5686                        (match_operand:MVE_2 3 "s_register_operand" "w")
5687                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5688          VHADDQ_M))
5689   ]
5690   "TARGET_HAVE_MVE"
5691   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5692   [(set_attr "type" "mve_move")
5693    (set_attr "length""8")])
5694
5695 ;;
5696 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5697 ;;
5698 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5699   [
5700    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5701         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5702                        (match_operand:MVE_2 2 "s_register_operand" "w")
5703                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5704                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5705          VHSUBQ_M_N))
5706   ]
5707   "TARGET_HAVE_MVE"
5708   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5709   [(set_attr "type" "mve_move")
5710    (set_attr "length""8")])
5711
5712 ;;
5713 ;; [vhsubq_m_s, vhsubq_m_u])
5714 ;;
5715 (define_insn "mve_vhsubq_m_<supf><mode>"
5716   [
5717    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5718         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5719                        (match_operand:MVE_2 2 "s_register_operand" "w")
5720                        (match_operand:MVE_2 3 "s_register_operand" "w")
5721                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5722          VHSUBQ_M))
5723   ]
5724   "TARGET_HAVE_MVE"
5725   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5726   [(set_attr "type" "mve_move")
5727    (set_attr "length""8")])
5728
5729 ;;
5730 ;; [vmaxq_m_s, vmaxq_m_u])
5731 ;;
5732 (define_insn "mve_vmaxq_m_<supf><mode>"
5733   [
5734    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5735         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5736                        (match_operand:MVE_2 2 "s_register_operand" "w")
5737                        (match_operand:MVE_2 3 "s_register_operand" "w")
5738                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5739          VMAXQ_M))
5740   ]
5741   "TARGET_HAVE_MVE"
5742   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5743   [(set_attr "type" "mve_move")
5744    (set_attr "length""8")])
5745
5746 ;;
5747 ;; [vminq_m_s, vminq_m_u])
5748 ;;
5749 (define_insn "mve_vminq_m_<supf><mode>"
5750   [
5751    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5752         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5753                        (match_operand:MVE_2 2 "s_register_operand" "w")
5754                        (match_operand:MVE_2 3 "s_register_operand" "w")
5755                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5756          VMINQ_M))
5757   ]
5758   "TARGET_HAVE_MVE"
5759   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5760   [(set_attr "type" "mve_move")
5761    (set_attr "length""8")])
5762
5763 ;;
5764 ;; [vmladavaq_p_u, vmladavaq_p_s])
5765 ;;
5766 (define_insn "mve_vmladavaq_p_<supf><mode>"
5767   [
5768    (set (match_operand:SI 0 "s_register_operand" "=Te")
5769         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5770                     (match_operand:MVE_2 2 "s_register_operand" "w")
5771                     (match_operand:MVE_2 3 "s_register_operand" "w")
5772                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5773          VMLADAVAQ_P))
5774   ]
5775   "TARGET_HAVE_MVE"
5776   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5777   [(set_attr "type" "mve_move")
5778    (set_attr "length""8")])
5779
5780 ;;
5781 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5782 ;;
5783 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5784   [
5785    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5786         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5787                        (match_operand:MVE_2 2 "s_register_operand" "w")
5788                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5789                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5790          VMLAQ_M_N))
5791   ]
5792   "TARGET_HAVE_MVE"
5793   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5794   [(set_attr "type" "mve_move")
5795    (set_attr "length""8")])
5796
5797 ;;
5798 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5799 ;;
5800 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5801   [
5802    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5803         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5804                        (match_operand:MVE_2 2 "s_register_operand" "w")
5805                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5806                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5807          VMLASQ_M_N))
5808   ]
5809   "TARGET_HAVE_MVE"
5810   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5811   [(set_attr "type" "mve_move")
5812    (set_attr "length""8")])
5813
5814 ;;
5815 ;; [vmulhq_m_s, vmulhq_m_u])
5816 ;;
5817 (define_insn "mve_vmulhq_m_<supf><mode>"
5818   [
5819    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5820         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5821                        (match_operand:MVE_2 2 "s_register_operand" "w")
5822                        (match_operand:MVE_2 3 "s_register_operand" "w")
5823                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5824          VMULHQ_M))
5825   ]
5826   "TARGET_HAVE_MVE"
5827   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5828   [(set_attr "type" "mve_move")
5829    (set_attr "length""8")])
5830
5831 ;;
5832 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5833 ;;
5834 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5835   [
5836    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5837         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5838                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5839                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5840                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5841          VMULLBQ_INT_M))
5842   ]
5843   "TARGET_HAVE_MVE"
5844   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5845   [(set_attr "type" "mve_move")
5846    (set_attr "length""8")])
5847
5848 ;;
5849 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5850 ;;
5851 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5852   [
5853    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5854         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5855                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5856                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5857                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5858          VMULLTQ_INT_M))
5859   ]
5860   "TARGET_HAVE_MVE"
5861   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5862   [(set_attr "type" "mve_move")
5863    (set_attr "length""8")])
5864
5865 ;;
5866 ;; [vmulq_m_n_u, vmulq_m_n_s])
5867 ;;
5868 (define_insn "mve_vmulq_m_n_<supf><mode>"
5869   [
5870    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5871         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5872                        (match_operand:MVE_2 2 "s_register_operand" "w")
5873                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5874                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5875          VMULQ_M_N))
5876   ]
5877   "TARGET_HAVE_MVE"
5878   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5879   [(set_attr "type" "mve_move")
5880    (set_attr "length""8")])
5881
5882 ;;
5883 ;; [vmulq_m_s, vmulq_m_u])
5884 ;;
5885 (define_insn "mve_vmulq_m_<supf><mode>"
5886   [
5887    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5888         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5889                        (match_operand:MVE_2 2 "s_register_operand" "w")
5890                        (match_operand:MVE_2 3 "s_register_operand" "w")
5891                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5892          VMULQ_M))
5893   ]
5894   "TARGET_HAVE_MVE"
5895   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5896   [(set_attr "type" "mve_move")
5897    (set_attr "length""8")])
5898
5899 ;;
5900 ;; [vornq_m_u, vornq_m_s])
5901 ;;
5902 (define_insn "mve_vornq_m_<supf><mode>"
5903   [
5904    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5905         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5906                        (match_operand:MVE_2 2 "s_register_operand" "w")
5907                        (match_operand:MVE_2 3 "s_register_operand" "w")
5908                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5909          VORNQ_M))
5910   ]
5911   "TARGET_HAVE_MVE"
5912   "vpst\;vornt %q0, %q2, %q3"
5913   [(set_attr "type" "mve_move")
5914    (set_attr "length""8")])
5915
5916 ;;
5917 ;; [vorrq_m_s, vorrq_m_u])
5918 ;;
5919 (define_insn "mve_vorrq_m_<supf><mode>"
5920   [
5921    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5922         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5923                        (match_operand:MVE_2 2 "s_register_operand" "w")
5924                        (match_operand:MVE_2 3 "s_register_operand" "w")
5925                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5926          VORRQ_M))
5927   ]
5928   "TARGET_HAVE_MVE"
5929   "vpst\;vorrt %q0, %q2, %q3"
5930   [(set_attr "type" "mve_move")
5931    (set_attr "length""8")])
5932
5933 ;;
5934 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5935 ;;
5936 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5937   [
5938    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5939         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5940                        (match_operand:MVE_2 2 "s_register_operand" "w")
5941                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5942                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5943          VQADDQ_M_N))
5944   ]
5945   "TARGET_HAVE_MVE"
5946   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5947   [(set_attr "type" "mve_move")
5948    (set_attr "length""8")])
5949
5950 ;;
5951 ;; [vqaddq_m_u, vqaddq_m_s])
5952 ;;
5953 (define_insn "mve_vqaddq_m_<supf><mode>"
5954   [
5955    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5956         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5957                        (match_operand:MVE_2 2 "s_register_operand" "w")
5958                        (match_operand:MVE_2 3 "s_register_operand" "w")
5959                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5960          VQADDQ_M))
5961   ]
5962   "TARGET_HAVE_MVE"
5963   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5964   [(set_attr "type" "mve_move")
5965    (set_attr "length""8")])
5966
5967 ;;
5968 ;; [vqdmlahq_m_n_s])
5969 ;;
5970 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5971   [
5972    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5973         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5974                        (match_operand:MVE_2 2 "s_register_operand" "w")
5975                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5976                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5977          VQDMLAHQ_M_N_S))
5978   ]
5979   "TARGET_HAVE_MVE"
5980   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5981   [(set_attr "type" "mve_move")
5982    (set_attr "length""8")])
5983
5984 ;;
5985 ;; [vqdmlashq_m_n_s])
5986 ;;
5987 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5988   [
5989    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5990         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5991                        (match_operand:MVE_2 2 "s_register_operand" "w")
5992                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5993                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5994          VQDMLASHQ_M_N_S))
5995   ]
5996   "TARGET_HAVE_MVE"
5997   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5998   [(set_attr "type" "mve_move")
5999    (set_attr "length""8")])
6000
6001 ;;
6002 ;; [vqrdmlahq_m_n_s])
6003 ;;
6004 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6005   [
6006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6008                        (match_operand:MVE_2 2 "s_register_operand" "w")
6009                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6010                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6011          VQRDMLAHQ_M_N_S))
6012   ]
6013   "TARGET_HAVE_MVE"
6014   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6015   [(set_attr "type" "mve_move")
6016    (set_attr "length""8")])
6017
6018 ;;
6019 ;; [vqrdmlashq_m_n_s])
6020 ;;
6021 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6022   [
6023    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6024         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6025                        (match_operand:MVE_2 2 "s_register_operand" "w")
6026                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6027                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6028          VQRDMLASHQ_M_N_S))
6029   ]
6030   "TARGET_HAVE_MVE"
6031   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6032   [(set_attr "type" "mve_move")
6033    (set_attr "length""8")])
6034
6035 ;;
6036 ;; [vqrshlq_m_u, vqrshlq_m_s])
6037 ;;
6038 (define_insn "mve_vqrshlq_m_<supf><mode>"
6039   [
6040    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6041         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6042                        (match_operand:MVE_2 2 "s_register_operand" "w")
6043                        (match_operand:MVE_2 3 "s_register_operand" "w")
6044                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6045          VQRSHLQ_M))
6046   ]
6047   "TARGET_HAVE_MVE"
6048   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6049   [(set_attr "type" "mve_move")
6050    (set_attr "length""8")])
6051
6052 ;;
6053 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6054 ;;
6055 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6056   [
6057    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6058         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6059                        (match_operand:MVE_2 2 "s_register_operand" "w")
6060                        (match_operand:SI 3 "immediate_operand" "i")
6061                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6062          VQSHLQ_M_N))
6063   ]
6064   "TARGET_HAVE_MVE"
6065   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6066   [(set_attr "type" "mve_move")
6067    (set_attr "length""8")])
6068
6069 ;;
6070 ;; [vqshlq_m_u, vqshlq_m_s])
6071 ;;
6072 (define_insn "mve_vqshlq_m_<supf><mode>"
6073   [
6074    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6075         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6076                        (match_operand:MVE_2 2 "s_register_operand" "w")
6077                        (match_operand:MVE_2 3 "s_register_operand" "w")
6078                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6079          VQSHLQ_M))
6080   ]
6081   "TARGET_HAVE_MVE"
6082   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6083   [(set_attr "type" "mve_move")
6084    (set_attr "length""8")])
6085
6086 ;;
6087 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6088 ;;
6089 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6090   [
6091    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6092         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6093                        (match_operand:MVE_2 2 "s_register_operand" "w")
6094                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6095                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6096          VQSUBQ_M_N))
6097   ]
6098   "TARGET_HAVE_MVE"
6099   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6100   [(set_attr "type" "mve_move")
6101    (set_attr "length""8")])
6102
6103 ;;
6104 ;; [vqsubq_m_u, vqsubq_m_s])
6105 ;;
6106 (define_insn "mve_vqsubq_m_<supf><mode>"
6107   [
6108    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6109         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6110                        (match_operand:MVE_2 2 "s_register_operand" "w")
6111                        (match_operand:MVE_2 3 "s_register_operand" "w")
6112                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6113          VQSUBQ_M))
6114   ]
6115   "TARGET_HAVE_MVE"
6116   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6117   [(set_attr "type" "mve_move")
6118    (set_attr "length""8")])
6119
6120 ;;
6121 ;; [vrhaddq_m_u, vrhaddq_m_s])
6122 ;;
6123 (define_insn "mve_vrhaddq_m_<supf><mode>"
6124   [
6125    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6126         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6127                        (match_operand:MVE_2 2 "s_register_operand" "w")
6128                        (match_operand:MVE_2 3 "s_register_operand" "w")
6129                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6130          VRHADDQ_M))
6131   ]
6132   "TARGET_HAVE_MVE"
6133   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6134   [(set_attr "type" "mve_move")
6135    (set_attr "length""8")])
6136
6137 ;;
6138 ;; [vrmulhq_m_u, vrmulhq_m_s])
6139 ;;
6140 (define_insn "mve_vrmulhq_m_<supf><mode>"
6141   [
6142    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6143         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6144                        (match_operand:MVE_2 2 "s_register_operand" "w")
6145                        (match_operand:MVE_2 3 "s_register_operand" "w")
6146                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6147          VRMULHQ_M))
6148   ]
6149   "TARGET_HAVE_MVE"
6150   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6151   [(set_attr "type" "mve_move")
6152    (set_attr "length""8")])
6153
6154 ;;
6155 ;; [vrshlq_m_s, vrshlq_m_u])
6156 ;;
6157 (define_insn "mve_vrshlq_m_<supf><mode>"
6158   [
6159    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6160         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6161                        (match_operand:MVE_2 2 "s_register_operand" "w")
6162                        (match_operand:MVE_2 3 "s_register_operand" "w")
6163                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6164          VRSHLQ_M))
6165   ]
6166   "TARGET_HAVE_MVE"
6167   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6168   [(set_attr "type" "mve_move")
6169    (set_attr "length""8")])
6170
6171 ;;
6172 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6173 ;;
6174 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6175   [
6176    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6177         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6178                        (match_operand:MVE_2 2 "s_register_operand" "w")
6179                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6180                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6181          VRSHRQ_M_N))
6182   ]
6183   "TARGET_HAVE_MVE"
6184   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6185   [(set_attr "type" "mve_move")
6186    (set_attr "length""8")])
6187
6188 ;;
6189 ;; [vshlq_m_n_s, vshlq_m_n_u])
6190 ;;
6191 (define_insn "mve_vshlq_m_n_<supf><mode>"
6192   [
6193    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6194         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6195                        (match_operand:MVE_2 2 "s_register_operand" "w")
6196                        (match_operand:SI 3 "immediate_operand" "i")
6197                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6198          VSHLQ_M_N))
6199   ]
6200   "TARGET_HAVE_MVE"
6201   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6202   [(set_attr "type" "mve_move")
6203    (set_attr "length""8")])
6204
6205 ;;
6206 ;; [vshrq_m_n_s, vshrq_m_n_u])
6207 ;;
6208 (define_insn "mve_vshrq_m_n_<supf><mode>"
6209   [
6210    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6211         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6212                        (match_operand:MVE_2 2 "s_register_operand" "w")
6213                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6214                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6215          VSHRQ_M_N))
6216   ]
6217   "TARGET_HAVE_MVE"
6218   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6219   [(set_attr "type" "mve_move")
6220    (set_attr "length""8")])
6221
6222 ;;
6223 ;; [vsliq_m_n_u, vsliq_m_n_s])
6224 ;;
6225 (define_insn "mve_vsliq_m_n_<supf><mode>"
6226    [
6227    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6228        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6229                        (match_operand:MVE_2 2 "s_register_operand" "w")
6230                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6231                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6232          VSLIQ_M_N))
6233   ]
6234   "TARGET_HAVE_MVE"
6235   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6236   [(set_attr "type" "mve_move")
6237    (set_attr "length""8")])
6238
6239 ;;
6240 ;; [vsubq_m_n_s, vsubq_m_n_u])
6241 ;;
6242 (define_insn "mve_vsubq_m_n_<supf><mode>"
6243   [
6244    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6245         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6246                        (match_operand:MVE_2 2 "s_register_operand" "w")
6247                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6248                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6249          VSUBQ_M_N))
6250   ]
6251   "TARGET_HAVE_MVE"
6252   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6253   [(set_attr "type" "mve_move")
6254    (set_attr "length""8")])
6255
6256 ;;
6257 ;; [vhcaddq_rot270_m_s])
6258 ;;
6259 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6260   [
6261    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6262         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6263                        (match_operand:MVE_2 2 "s_register_operand" "w")
6264                        (match_operand:MVE_2 3 "s_register_operand" "w")
6265                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6266          VHCADDQ_ROT270_M_S))
6267   ]
6268   "TARGET_HAVE_MVE"
6269   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6270   [(set_attr "type" "mve_move")
6271    (set_attr "length""8")])
6272
6273 ;;
6274 ;; [vhcaddq_rot90_m_s])
6275 ;;
6276 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6277   [
6278    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6279         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6280                        (match_operand:MVE_2 2 "s_register_operand" "w")
6281                        (match_operand:MVE_2 3 "s_register_operand" "w")
6282                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6283          VHCADDQ_ROT90_M_S))
6284   ]
6285   "TARGET_HAVE_MVE"
6286   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6287   [(set_attr "type" "mve_move")
6288    (set_attr "length""8")])
6289
6290 ;;
6291 ;; [vmladavaxq_p_s])
6292 ;;
6293 (define_insn "mve_vmladavaxq_p_s<mode>"
6294   [
6295    (set (match_operand:SI 0 "s_register_operand" "=Te")
6296         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6297                        (match_operand:MVE_2 2 "s_register_operand" "w")
6298                        (match_operand:MVE_2 3 "s_register_operand" "w")
6299                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6300          VMLADAVAXQ_P_S))
6301   ]
6302   "TARGET_HAVE_MVE"
6303   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6304   [(set_attr "type" "mve_move")
6305    (set_attr "length""8")])
6306
6307 ;;
6308 ;; [vmlsdavaq_p_s])
6309 ;;
6310 (define_insn "mve_vmlsdavaq_p_s<mode>"
6311   [
6312    (set (match_operand:SI 0 "s_register_operand" "=Te")
6313         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6314                        (match_operand:MVE_2 2 "s_register_operand" "w")
6315                        (match_operand:MVE_2 3 "s_register_operand" "w")
6316                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6317          VMLSDAVAQ_P_S))
6318   ]
6319   "TARGET_HAVE_MVE"
6320   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6321   [(set_attr "type" "mve_move")
6322    (set_attr "length""8")])
6323
6324 ;;
6325 ;; [vmlsdavaxq_p_s])
6326 ;;
6327 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6328   [
6329    (set (match_operand:SI 0 "s_register_operand" "=Te")
6330         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6331                        (match_operand:MVE_2 2 "s_register_operand" "w")
6332                        (match_operand:MVE_2 3 "s_register_operand" "w")
6333                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6334          VMLSDAVAXQ_P_S))
6335   ]
6336   "TARGET_HAVE_MVE"
6337   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6338   [(set_attr "type" "mve_move")
6339    (set_attr "length""8")])
6340
6341 ;;
6342 ;; [vqdmladhq_m_s])
6343 ;;
6344 (define_insn "mve_vqdmladhq_m_s<mode>"
6345   [
6346    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6347         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6348                        (match_operand:MVE_2 2 "s_register_operand" "w")
6349                        (match_operand:MVE_2 3 "s_register_operand" "w")
6350                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6351          VQDMLADHQ_M_S))
6352   ]
6353   "TARGET_HAVE_MVE"
6354   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6355   [(set_attr "type" "mve_move")
6356    (set_attr "length""8")])
6357
6358 ;;
6359 ;; [vqdmladhxq_m_s])
6360 ;;
6361 (define_insn "mve_vqdmladhxq_m_s<mode>"
6362   [
6363    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6364         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6365                        (match_operand:MVE_2 2 "s_register_operand" "w")
6366                        (match_operand:MVE_2 3 "s_register_operand" "w")
6367                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6368          VQDMLADHXQ_M_S))
6369   ]
6370   "TARGET_HAVE_MVE"
6371   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6372   [(set_attr "type" "mve_move")
6373    (set_attr "length""8")])
6374
6375 ;;
6376 ;; [vqdmlsdhq_m_s])
6377 ;;
6378 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6379   [
6380    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6381         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6382                        (match_operand:MVE_2 2 "s_register_operand" "w")
6383                        (match_operand:MVE_2 3 "s_register_operand" "w")
6384                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6385          VQDMLSDHQ_M_S))
6386   ]
6387   "TARGET_HAVE_MVE"
6388   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6389   [(set_attr "type" "mve_move")
6390    (set_attr "length""8")])
6391
6392 ;;
6393 ;; [vqdmlsdhxq_m_s])
6394 ;;
6395 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6396   [
6397    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6398         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6399                        (match_operand:MVE_2 2 "s_register_operand" "w")
6400                        (match_operand:MVE_2 3 "s_register_operand" "w")
6401                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6402          VQDMLSDHXQ_M_S))
6403   ]
6404   "TARGET_HAVE_MVE"
6405   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6406   [(set_attr "type" "mve_move")
6407    (set_attr "length""8")])
6408
6409 ;;
6410 ;; [vqdmulhq_m_n_s])
6411 ;;
6412 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6413   [
6414    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6415         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6416                        (match_operand:MVE_2 2 "s_register_operand" "w")
6417                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6418                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6419          VQDMULHQ_M_N_S))
6420   ]
6421   "TARGET_HAVE_MVE"
6422   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6423   [(set_attr "type" "mve_move")
6424    (set_attr "length""8")])
6425
6426 ;;
6427 ;; [vqdmulhq_m_s])
6428 ;;
6429 (define_insn "mve_vqdmulhq_m_s<mode>"
6430   [
6431    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6432         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6433                        (match_operand:MVE_2 2 "s_register_operand" "w")
6434                        (match_operand:MVE_2 3 "s_register_operand" "w")
6435                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6436          VQDMULHQ_M_S))
6437   ]
6438   "TARGET_HAVE_MVE"
6439   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6440   [(set_attr "type" "mve_move")
6441    (set_attr "length""8")])
6442
6443 ;;
6444 ;; [vqrdmladhq_m_s])
6445 ;;
6446 (define_insn "mve_vqrdmladhq_m_s<mode>"
6447   [
6448    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6449         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6450                        (match_operand:MVE_2 2 "s_register_operand" "w")
6451                        (match_operand:MVE_2 3 "s_register_operand" "w")
6452                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6453          VQRDMLADHQ_M_S))
6454   ]
6455   "TARGET_HAVE_MVE"
6456   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6457   [(set_attr "type" "mve_move")
6458    (set_attr "length""8")])
6459
6460 ;;
6461 ;; [vqrdmladhxq_m_s])
6462 ;;
6463 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6464   [
6465    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6466         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6467                        (match_operand:MVE_2 2 "s_register_operand" "w")
6468                        (match_operand:MVE_2 3 "s_register_operand" "w")
6469                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6470          VQRDMLADHXQ_M_S))
6471   ]
6472   "TARGET_HAVE_MVE"
6473   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6474   [(set_attr "type" "mve_move")
6475    (set_attr "length""8")])
6476
6477 ;;
6478 ;; [vqrdmlsdhq_m_s])
6479 ;;
6480 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6481   [
6482    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6483         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6484                        (match_operand:MVE_2 2 "s_register_operand" "w")
6485                        (match_operand:MVE_2 3 "s_register_operand" "w")
6486                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6487          VQRDMLSDHQ_M_S))
6488   ]
6489   "TARGET_HAVE_MVE"
6490   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6491   [(set_attr "type" "mve_move")
6492    (set_attr "length""8")])
6493
6494 ;;
6495 ;; [vqrdmlsdhxq_m_s])
6496 ;;
6497 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6498   [
6499    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6500         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6501                        (match_operand:MVE_2 2 "s_register_operand" "w")
6502                        (match_operand:MVE_2 3 "s_register_operand" "w")
6503                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6504          VQRDMLSDHXQ_M_S))
6505   ]
6506   "TARGET_HAVE_MVE"
6507   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6508   [(set_attr "type" "mve_move")
6509    (set_attr "length""8")])
6510
6511 ;;
6512 ;; [vqrdmulhq_m_n_s])
6513 ;;
6514 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6515   [
6516    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6517         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6518                        (match_operand:MVE_2 2 "s_register_operand" "w")
6519                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6520                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6521          VQRDMULHQ_M_N_S))
6522   ]
6523   "TARGET_HAVE_MVE"
6524   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6525   [(set_attr "type" "mve_move")
6526    (set_attr "length""8")])
6527
6528 ;;
6529 ;; [vqrdmulhq_m_s])
6530 ;;
6531 (define_insn "mve_vqrdmulhq_m_s<mode>"
6532   [
6533    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6534         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6535                        (match_operand:MVE_2 2 "s_register_operand" "w")
6536                        (match_operand:MVE_2 3 "s_register_operand" "w")
6537                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6538          VQRDMULHQ_M_S))
6539   ]
6540   "TARGET_HAVE_MVE"
6541   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6542   [(set_attr "type" "mve_move")
6543    (set_attr "length""8")])
6544
6545 ;;
6546 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6547 ;;
6548 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6549   [
6550    (set (match_operand:DI 0 "s_register_operand" "=r")
6551         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6552                        (match_operand:MVE_5 2 "s_register_operand" "w")
6553                        (match_operand:MVE_5 3 "s_register_operand" "w")
6554                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6555          VMLALDAVAQ_P))
6556   ]
6557   "TARGET_HAVE_MVE"
6558   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6559   [(set_attr "type" "mve_move")
6560    (set_attr "length""8")])
6561
6562 ;;
6563 ;; [vmlaldavaxq_p_s])
6564 ;;
6565 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6566   [
6567    (set (match_operand:DI 0 "s_register_operand" "=r")
6568         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6569                        (match_operand:MVE_5 2 "s_register_operand" "w")
6570                        (match_operand:MVE_5 3 "s_register_operand" "w")
6571                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6572          VMLALDAVAXQ_P))
6573   ]
6574   "TARGET_HAVE_MVE"
6575   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6576   [(set_attr "type" "mve_move")
6577    (set_attr "length""8")])
6578
6579 ;;
6580 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6581 ;;
6582 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6583   [
6584    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6585         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6586                        (match_operand:MVE_5 2 "s_register_operand" "w")
6587                        (match_operand:SI 3 "mve_imm_8" "Rb")
6588                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6589          VQRSHRNBQ_M_N))
6590   ]
6591   "TARGET_HAVE_MVE"
6592   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6593   [(set_attr "type" "mve_move")
6594    (set_attr "length""8")])
6595
6596 ;;
6597 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6598 ;;
6599 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6600   [
6601    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6602         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6603                        (match_operand:MVE_5 2 "s_register_operand" "w")
6604                        (match_operand:SI 3 "mve_imm_8" "Rb")
6605                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6606          VQRSHRNTQ_M_N))
6607   ]
6608   "TARGET_HAVE_MVE"
6609   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6610   [(set_attr "type" "mve_move")
6611    (set_attr "length""8")])
6612
6613 ;;
6614 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6615 ;;
6616 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6617   [
6618    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6619         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6620                        (match_operand:MVE_5 2 "s_register_operand" "w")
6621                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6622                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6623          VQSHRNBQ_M_N))
6624   ]
6625   "TARGET_HAVE_MVE"
6626   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6627   [(set_attr "type" "mve_move")
6628    (set_attr "length""8")])
6629
6630 ;;
6631 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6632 ;;
6633 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6634   [
6635    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6636         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6637                        (match_operand:MVE_5 2 "s_register_operand" "w")
6638                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6639                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6640          VQSHRNTQ_M_N))
6641   ]
6642   "TARGET_HAVE_MVE"
6643   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6644   [(set_attr "type" "mve_move")
6645    (set_attr "length""8")])
6646
6647 ;;
6648 ;; [vrmlaldavhaq_p_s])
6649 ;;
6650 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6651   [
6652    (set (match_operand:DI 0 "s_register_operand" "=r")
6653         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6654                        (match_operand:V4SI 2 "s_register_operand" "w")
6655                        (match_operand:V4SI 3 "s_register_operand" "w")
6656                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6657          VRMLALDAVHAQ_P_S))
6658   ]
6659   "TARGET_HAVE_MVE"
6660   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6661   [(set_attr "type" "mve_move")
6662    (set_attr "length""8")])
6663
6664 ;;
6665 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6666 ;;
6667 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6668   [
6669    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6670         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6671                        (match_operand:MVE_5 2 "s_register_operand" "w")
6672                        (match_operand:SI 3 "mve_imm_8" "Rb")
6673                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6674          VRSHRNBQ_M_N))
6675   ]
6676   "TARGET_HAVE_MVE"
6677   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6678   [(set_attr "type" "mve_move")
6679    (set_attr "length""8")])
6680
6681 ;;
6682 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6683 ;;
6684 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6685   [
6686    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6687         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6688                        (match_operand:MVE_5 2 "s_register_operand" "w")
6689                        (match_operand:SI 3 "mve_imm_8" "Rb")
6690                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6691          VRSHRNTQ_M_N))
6692   ]
6693   "TARGET_HAVE_MVE"
6694   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6695   [(set_attr "type" "mve_move")
6696    (set_attr "length""8")])
6697
6698 ;;
6699 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6700 ;;
6701 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6702   [
6703    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6704         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6705                        (match_operand:MVE_3 2 "s_register_operand" "w")
6706                        (match_operand:SI 3 "immediate_operand" "i")
6707                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6708          VSHLLBQ_M_N))
6709   ]
6710   "TARGET_HAVE_MVE"
6711   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6712   [(set_attr "type" "mve_move")
6713    (set_attr "length""8")])
6714
6715 ;;
6716 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6717 ;;
6718 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6719   [
6720    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6721         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6722                        (match_operand:MVE_3 2 "s_register_operand" "w")
6723                        (match_operand:SI 3 "immediate_operand" "i")
6724                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6725          VSHLLTQ_M_N))
6726   ]
6727   "TARGET_HAVE_MVE"
6728   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6729   [(set_attr "type" "mve_move")
6730    (set_attr "length""8")])
6731
6732 ;;
6733 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6734 ;;
6735 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6736   [
6737    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6738         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6739                        (match_operand:MVE_5 2 "s_register_operand" "w")
6740                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6741                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6742          VSHRNBQ_M_N))
6743   ]
6744   "TARGET_HAVE_MVE"
6745   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6746   [(set_attr "type" "mve_move")
6747    (set_attr "length""8")])
6748
6749 ;;
6750 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6751 ;;
6752 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6753   [
6754    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6755         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6756                        (match_operand:MVE_5 2 "s_register_operand" "w")
6757                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6758                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6759          VSHRNTQ_M_N))
6760   ]
6761   "TARGET_HAVE_MVE"
6762   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6763   [(set_attr "type" "mve_move")
6764    (set_attr "length""8")])
6765
6766 ;;
6767 ;; [vmlsldavaq_p_s])
6768 ;;
6769 (define_insn "mve_vmlsldavaq_p_s<mode>"
6770   [
6771    (set (match_operand:DI 0 "s_register_operand" "=r")
6772         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6773                        (match_operand:MVE_5 2 "s_register_operand" "w")
6774                        (match_operand:MVE_5 3 "s_register_operand" "w")
6775                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6776          VMLSLDAVAQ_P_S))
6777   ]
6778   "TARGET_HAVE_MVE"
6779   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6780   [(set_attr "type" "mve_move")
6781    (set_attr "length""8")])
6782
6783 ;;
6784 ;; [vmlsldavaxq_p_s])
6785 ;;
6786 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6787   [
6788    (set (match_operand:DI 0 "s_register_operand" "=r")
6789         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6790                        (match_operand:MVE_5 2 "s_register_operand" "w")
6791                        (match_operand:MVE_5 3 "s_register_operand" "w")
6792                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6793          VMLSLDAVAXQ_P_S))
6794   ]
6795   "TARGET_HAVE_MVE"
6796   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6797   [(set_attr "type" "mve_move")
6798    (set_attr "length""8")])
6799
6800 ;;
6801 ;; [vmullbq_poly_m_p])
6802 ;;
6803 (define_insn "mve_vmullbq_poly_m_p<mode>"
6804   [
6805    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6806         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6807                        (match_operand:MVE_3 2 "s_register_operand" "w")
6808                        (match_operand:MVE_3 3 "s_register_operand" "w")
6809                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6810          VMULLBQ_POLY_M_P))
6811   ]
6812   "TARGET_HAVE_MVE"
6813   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6814   [(set_attr "type" "mve_move")
6815    (set_attr "length""8")])
6816
6817 ;;
6818 ;; [vmulltq_poly_m_p])
6819 ;;
6820 (define_insn "mve_vmulltq_poly_m_p<mode>"
6821   [
6822    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6823         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6824                        (match_operand:MVE_3 2 "s_register_operand" "w")
6825                        (match_operand:MVE_3 3 "s_register_operand" "w")
6826                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6827          VMULLTQ_POLY_M_P))
6828   ]
6829   "TARGET_HAVE_MVE"
6830   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6831   [(set_attr "type" "mve_move")
6832    (set_attr "length""8")])
6833
6834 ;;
6835 ;; [vqdmullbq_m_n_s])
6836 ;;
6837 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6838   [
6839    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6840         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6841                        (match_operand:MVE_5 2 "s_register_operand" "w")
6842                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6843                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6844          VQDMULLBQ_M_N_S))
6845   ]
6846   "TARGET_HAVE_MVE"
6847   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6848   [(set_attr "type" "mve_move")
6849    (set_attr "length""8")])
6850
6851 ;;
6852 ;; [vqdmullbq_m_s])
6853 ;;
6854 (define_insn "mve_vqdmullbq_m_s<mode>"
6855   [
6856    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6857         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6858                        (match_operand:MVE_5 2 "s_register_operand" "w")
6859                        (match_operand:MVE_5 3 "s_register_operand" "w")
6860                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6861          VQDMULLBQ_M_S))
6862   ]
6863   "TARGET_HAVE_MVE"
6864   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6865   [(set_attr "type" "mve_move")
6866    (set_attr "length""8")])
6867
6868 ;;
6869 ;; [vqdmulltq_m_n_s])
6870 ;;
6871 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6872   [
6873    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6874         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6875                        (match_operand:MVE_5 2 "s_register_operand" "w")
6876                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6877                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6878          VQDMULLTQ_M_N_S))
6879   ]
6880   "TARGET_HAVE_MVE"
6881   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6882   [(set_attr "type" "mve_move")
6883    (set_attr "length""8")])
6884
6885 ;;
6886 ;; [vqdmulltq_m_s])
6887 ;;
6888 (define_insn "mve_vqdmulltq_m_s<mode>"
6889   [
6890    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6891         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6892                        (match_operand:MVE_5 2 "s_register_operand" "w")
6893                        (match_operand:MVE_5 3 "s_register_operand" "w")
6894                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6895          VQDMULLTQ_M_S))
6896   ]
6897   "TARGET_HAVE_MVE"
6898   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6899   [(set_attr "type" "mve_move")
6900    (set_attr "length""8")])
6901
6902 ;;
6903 ;; [vqrshrunbq_m_n_s])
6904 ;;
6905 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6906   [
6907    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6908         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6909                        (match_operand:MVE_5 2 "s_register_operand" "w")
6910                        (match_operand:SI 3 "mve_imm_8" "Rb")
6911                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6912          VQRSHRUNBQ_M_N_S))
6913   ]
6914   "TARGET_HAVE_MVE"
6915   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6916   [(set_attr "type" "mve_move")
6917    (set_attr "length""8")])
6918
6919 ;;
6920 ;; [vqrshruntq_m_n_s])
6921 ;;
6922 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6923   [
6924    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6925         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6926                        (match_operand:MVE_5 2 "s_register_operand" "w")
6927                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6928                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6929          VQRSHRUNTQ_M_N_S))
6930   ]
6931   "TARGET_HAVE_MVE"
6932   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6933   [(set_attr "type" "mve_move")
6934    (set_attr "length""8")])
6935
6936 ;;
6937 ;; [vqshrunbq_m_n_s])
6938 ;;
6939 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6940   [
6941    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6942         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6943                        (match_operand:MVE_5 2 "s_register_operand" "w")
6944                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6945                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6946          VQSHRUNBQ_M_N_S))
6947   ]
6948   "TARGET_HAVE_MVE"
6949   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6950   [(set_attr "type" "mve_move")
6951    (set_attr "length""8")])
6952
6953 ;;
6954 ;; [vqshruntq_m_n_s])
6955 ;;
6956 (define_insn "mve_vqshruntq_m_n_s<mode>"
6957   [
6958    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6959         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6960                        (match_operand:MVE_5 2 "s_register_operand" "w")
6961                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6962                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6963          VQSHRUNTQ_M_N_S))
6964   ]
6965   "TARGET_HAVE_MVE"
6966   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6967   [(set_attr "type" "mve_move")
6968    (set_attr "length""8")])
6969
6970 ;;
6971 ;; [vrmlaldavhaq_p_u])
6972 ;;
6973 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6974   [
6975    (set (match_operand:DI 0 "s_register_operand" "=r")
6976         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6977                        (match_operand:V4SI 2 "s_register_operand" "w")
6978                        (match_operand:V4SI 3 "s_register_operand" "w")
6979                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6980          VRMLALDAVHAQ_P_U))
6981   ]
6982   "TARGET_HAVE_MVE"
6983   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6984   [(set_attr "type" "mve_move")
6985    (set_attr "length""8")])
6986
6987 ;;
6988 ;; [vrmlaldavhaxq_p_s])
6989 ;;
6990 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6991   [
6992    (set (match_operand:DI 0 "s_register_operand" "=r")
6993         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6994                        (match_operand:V4SI 2 "s_register_operand" "w")
6995                        (match_operand:V4SI 3 "s_register_operand" "w")
6996                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6997          VRMLALDAVHAXQ_P_S))
6998   ]
6999   "TARGET_HAVE_MVE"
7000   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7001   [(set_attr "type" "mve_move")
7002    (set_attr "length""8")])
7003
7004 ;;
7005 ;; [vrmlsldavhaq_p_s])
7006 ;;
7007 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7008   [
7009    (set (match_operand:DI 0 "s_register_operand" "=r")
7010         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7011                        (match_operand:V4SI 2 "s_register_operand" "w")
7012                        (match_operand:V4SI 3 "s_register_operand" "w")
7013                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7014          VRMLSLDAVHAQ_P_S))
7015   ]
7016   "TARGET_HAVE_MVE"
7017   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7018   [(set_attr "type" "mve_move")
7019    (set_attr "length""8")])
7020
7021 ;;
7022 ;; [vrmlsldavhaxq_p_s])
7023 ;;
7024 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7025   [
7026    (set (match_operand:DI 0 "s_register_operand" "=r")
7027         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7028                        (match_operand:V4SI 2 "s_register_operand" "w")
7029                        (match_operand:V4SI 3 "s_register_operand" "w")
7030                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7031          VRMLSLDAVHAXQ_P_S))
7032   ]
7033   "TARGET_HAVE_MVE"
7034   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7035   [(set_attr "type" "mve_move")
7036    (set_attr "length""8")])
7037 ;;
7038 ;; [vabdq_m_f])
7039 ;;
7040 (define_insn "mve_vabdq_m_f<mode>"
7041   [
7042    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7043         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7044                        (match_operand:MVE_0 2 "s_register_operand" "w")
7045                        (match_operand:MVE_0 3 "s_register_operand" "w")
7046                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7047          VABDQ_M_F))
7048   ]
7049   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7050   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7051   [(set_attr "type" "mve_move")
7052    (set_attr "length""8")])
7053
7054 ;;
7055 ;; [vaddq_m_f])
7056 ;;
7057 (define_insn "mve_vaddq_m_f<mode>"
7058   [
7059    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7060         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7061                        (match_operand:MVE_0 2 "s_register_operand" "w")
7062                        (match_operand:MVE_0 3 "s_register_operand" "w")
7063                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7064          VADDQ_M_F))
7065   ]
7066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7067   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7068   [(set_attr "type" "mve_move")
7069    (set_attr "length""8")])
7070
7071 ;;
7072 ;; [vaddq_m_n_f])
7073 ;;
7074 (define_insn "mve_vaddq_m_n_f<mode>"
7075   [
7076    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7077         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7078                        (match_operand:MVE_0 2 "s_register_operand" "w")
7079                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7080                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7081          VADDQ_M_N_F))
7082   ]
7083   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7084   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7085   [(set_attr "type" "mve_move")
7086    (set_attr "length""8")])
7087
7088 ;;
7089 ;; [vandq_m_f])
7090 ;;
7091 (define_insn "mve_vandq_m_f<mode>"
7092   [
7093    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7094         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7095                        (match_operand:MVE_0 2 "s_register_operand" "w")
7096                        (match_operand:MVE_0 3 "s_register_operand" "w")
7097                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7098          VANDQ_M_F))
7099   ]
7100   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7101   "vpst\;vandt %q0, %q2, %q3"
7102   [(set_attr "type" "mve_move")
7103    (set_attr "length""8")])
7104
7105 ;;
7106 ;; [vbicq_m_f])
7107 ;;
7108 (define_insn "mve_vbicq_m_f<mode>"
7109   [
7110    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7111         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7112                        (match_operand:MVE_0 2 "s_register_operand" "w")
7113                        (match_operand:MVE_0 3 "s_register_operand" "w")
7114                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7115          VBICQ_M_F))
7116   ]
7117   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7118   "vpst\;vbict %q0, %q2, %q3"
7119   [(set_attr "type" "mve_move")
7120    (set_attr "length""8")])
7121
7122 ;;
7123 ;; [vbrsrq_m_n_f])
7124 ;;
7125 (define_insn "mve_vbrsrq_m_n_f<mode>"
7126   [
7127    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7128         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7129                        (match_operand:MVE_0 2 "s_register_operand" "w")
7130                        (match_operand:SI 3 "s_register_operand" "r")
7131                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7132          VBRSRQ_M_N_F))
7133   ]
7134   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7135   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7136   [(set_attr "type" "mve_move")
7137    (set_attr "length""8")])
7138
7139 ;;
7140 ;; [vcaddq_rot270_m_f])
7141 ;;
7142 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7143   [
7144    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7145         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7146                        (match_operand:MVE_0 2 "s_register_operand" "w")
7147                        (match_operand:MVE_0 3 "s_register_operand" "w")
7148                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7149          VCADDQ_ROT270_M_F))
7150   ]
7151   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7152   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7153   [(set_attr "type" "mve_move")
7154    (set_attr "length""8")])
7155
7156 ;;
7157 ;; [vcaddq_rot90_m_f])
7158 ;;
7159 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7160   [
7161    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7162         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7163                        (match_operand:MVE_0 2 "s_register_operand" "w")
7164                        (match_operand:MVE_0 3 "s_register_operand" "w")
7165                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7166          VCADDQ_ROT90_M_F))
7167   ]
7168   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7169   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7170   [(set_attr "type" "mve_move")
7171    (set_attr "length""8")])
7172
7173 ;;
7174 ;; [vcmlaq_m_f])
7175 ;;
7176 (define_insn "mve_vcmlaq_m_f<mode>"
7177   [
7178    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7179         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7180                        (match_operand:MVE_0 2 "s_register_operand" "w")
7181                        (match_operand:MVE_0 3 "s_register_operand" "w")
7182                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7183          VCMLAQ_M_F))
7184   ]
7185   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7186   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7187   [(set_attr "type" "mve_move")
7188    (set_attr "length""8")])
7189
7190 ;;
7191 ;; [vcmlaq_rot180_m_f])
7192 ;;
7193 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7194   [
7195    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7196         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7197                        (match_operand:MVE_0 2 "s_register_operand" "w")
7198                        (match_operand:MVE_0 3 "s_register_operand" "w")
7199                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7200          VCMLAQ_ROT180_M_F))
7201   ]
7202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7203   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7204   [(set_attr "type" "mve_move")
7205    (set_attr "length""8")])
7206
7207 ;;
7208 ;; [vcmlaq_rot270_m_f])
7209 ;;
7210 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7211   [
7212    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7213         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7214                        (match_operand:MVE_0 2 "s_register_operand" "w")
7215                        (match_operand:MVE_0 3 "s_register_operand" "w")
7216                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7217          VCMLAQ_ROT270_M_F))
7218   ]
7219   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7220   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7221   [(set_attr "type" "mve_move")
7222    (set_attr "length""8")])
7223
7224 ;;
7225 ;; [vcmlaq_rot90_m_f])
7226 ;;
7227 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7228   [
7229    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7230         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7231                        (match_operand:MVE_0 2 "s_register_operand" "w")
7232                        (match_operand:MVE_0 3 "s_register_operand" "w")
7233                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7234          VCMLAQ_ROT90_M_F))
7235   ]
7236   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7237   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7238   [(set_attr "type" "mve_move")
7239    (set_attr "length""8")])
7240
7241 ;;
7242 ;; [vcmulq_m_f])
7243 ;;
7244 (define_insn "mve_vcmulq_m_f<mode>"
7245   [
7246    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7247         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7248                        (match_operand:MVE_0 2 "s_register_operand" "w")
7249                        (match_operand:MVE_0 3 "s_register_operand" "w")
7250                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7251          VCMULQ_M_F))
7252   ]
7253   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7254   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7255   [(set_attr "type" "mve_move")
7256    (set_attr "length""8")])
7257
7258 ;;
7259 ;; [vcmulq_rot180_m_f])
7260 ;;
7261 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7262   [
7263    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7264         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7265                        (match_operand:MVE_0 2 "s_register_operand" "w")
7266                        (match_operand:MVE_0 3 "s_register_operand" "w")
7267                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7268          VCMULQ_ROT180_M_F))
7269   ]
7270   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7271   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7272   [(set_attr "type" "mve_move")
7273    (set_attr "length""8")])
7274
7275 ;;
7276 ;; [vcmulq_rot270_m_f])
7277 ;;
7278 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7279   [
7280    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7281         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7282                        (match_operand:MVE_0 2 "s_register_operand" "w")
7283                        (match_operand:MVE_0 3 "s_register_operand" "w")
7284                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7285          VCMULQ_ROT270_M_F))
7286   ]
7287   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7288   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7289   [(set_attr "type" "mve_move")
7290    (set_attr "length""8")])
7291
7292 ;;
7293 ;; [vcmulq_rot90_m_f])
7294 ;;
7295 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7296   [
7297    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7298         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7299                        (match_operand:MVE_0 2 "s_register_operand" "w")
7300                        (match_operand:MVE_0 3 "s_register_operand" "w")
7301                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7302          VCMULQ_ROT90_M_F))
7303   ]
7304   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7305   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7306   [(set_attr "type" "mve_move")
7307    (set_attr "length""8")])
7308
7309 ;;
7310 ;; [veorq_m_f])
7311 ;;
7312 (define_insn "mve_veorq_m_f<mode>"
7313   [
7314    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7315         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7316                        (match_operand:MVE_0 2 "s_register_operand" "w")
7317                        (match_operand:MVE_0 3 "s_register_operand" "w")
7318                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7319          VEORQ_M_F))
7320   ]
7321   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7322   "vpst\;veort %q0, %q2, %q3"
7323   [(set_attr "type" "mve_move")
7324    (set_attr "length""8")])
7325
7326 ;;
7327 ;; [vfmaq_m_f])
7328 ;;
7329 (define_insn "mve_vfmaq_m_f<mode>"
7330   [
7331    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7332         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7333                        (match_operand:MVE_0 2 "s_register_operand" "w")
7334                        (match_operand:MVE_0 3 "s_register_operand" "w")
7335                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7336          VFMAQ_M_F))
7337   ]
7338   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7339   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7340   [(set_attr "type" "mve_move")
7341    (set_attr "length""8")])
7342
7343 ;;
7344 ;; [vfmaq_m_n_f])
7345 ;;
7346 (define_insn "mve_vfmaq_m_n_f<mode>"
7347   [
7348    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7349         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7350                        (match_operand:MVE_0 2 "s_register_operand" "w")
7351                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7352                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7353          VFMAQ_M_N_F))
7354   ]
7355   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7356   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7357   [(set_attr "type" "mve_move")
7358    (set_attr "length""8")])
7359
7360 ;;
7361 ;; [vfmasq_m_n_f])
7362 ;;
7363 (define_insn "mve_vfmasq_m_n_f<mode>"
7364   [
7365    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7366         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7367                        (match_operand:MVE_0 2 "s_register_operand" "w")
7368                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7369                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7370          VFMASQ_M_N_F))
7371   ]
7372   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7373   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7374   [(set_attr "type" "mve_move")
7375    (set_attr "length""8")])
7376
7377 ;;
7378 ;; [vfmsq_m_f])
7379 ;;
7380 (define_insn "mve_vfmsq_m_f<mode>"
7381   [
7382    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7383         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7384                        (match_operand:MVE_0 2 "s_register_operand" "w")
7385                        (match_operand:MVE_0 3 "s_register_operand" "w")
7386                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7387          VFMSQ_M_F))
7388   ]
7389   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7390   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7391   [(set_attr "type" "mve_move")
7392    (set_attr "length""8")])
7393
7394 ;;
7395 ;; [vmaxnmq_m_f])
7396 ;;
7397 (define_insn "mve_vmaxnmq_m_f<mode>"
7398   [
7399    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7400         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7401                        (match_operand:MVE_0 2 "s_register_operand" "w")
7402                        (match_operand:MVE_0 3 "s_register_operand" "w")
7403                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7404          VMAXNMQ_M_F))
7405   ]
7406   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7407   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7408   [(set_attr "type" "mve_move")
7409    (set_attr "length""8")])
7410
7411 ;;
7412 ;; [vminnmq_m_f])
7413 ;;
7414 (define_insn "mve_vminnmq_m_f<mode>"
7415   [
7416    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7417         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7418                        (match_operand:MVE_0 2 "s_register_operand" "w")
7419                        (match_operand:MVE_0 3 "s_register_operand" "w")
7420                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7421          VMINNMQ_M_F))
7422   ]
7423   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7424   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7425   [(set_attr "type" "mve_move")
7426    (set_attr "length""8")])
7427
7428 ;;
7429 ;; [vmulq_m_f])
7430 ;;
7431 (define_insn "mve_vmulq_m_f<mode>"
7432   [
7433    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7434         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7435                        (match_operand:MVE_0 2 "s_register_operand" "w")
7436                        (match_operand:MVE_0 3 "s_register_operand" "w")
7437                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7438          VMULQ_M_F))
7439   ]
7440   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7441   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7442   [(set_attr "type" "mve_move")
7443    (set_attr "length""8")])
7444
7445 ;;
7446 ;; [vmulq_m_n_f])
7447 ;;
7448 (define_insn "mve_vmulq_m_n_f<mode>"
7449   [
7450    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7451         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7452                        (match_operand:MVE_0 2 "s_register_operand" "w")
7453                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7454                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7455          VMULQ_M_N_F))
7456   ]
7457   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7458   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7459   [(set_attr "type" "mve_move")
7460    (set_attr "length""8")])
7461
7462 ;;
7463 ;; [vornq_m_f])
7464 ;;
7465 (define_insn "mve_vornq_m_f<mode>"
7466   [
7467    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7468         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7469                        (match_operand:MVE_0 2 "s_register_operand" "w")
7470                        (match_operand:MVE_0 3 "s_register_operand" "w")
7471                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7472          VORNQ_M_F))
7473   ]
7474   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7475   "vpst\;vornt %q0, %q2, %q3"
7476   [(set_attr "type" "mve_move")
7477    (set_attr "length""8")])
7478
7479 ;;
7480 ;; [vorrq_m_f])
7481 ;;
7482 (define_insn "mve_vorrq_m_f<mode>"
7483   [
7484    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7485         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7486                        (match_operand:MVE_0 2 "s_register_operand" "w")
7487                        (match_operand:MVE_0 3 "s_register_operand" "w")
7488                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7489          VORRQ_M_F))
7490   ]
7491   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7492   "vpst\;vorrt %q0, %q2, %q3"
7493   [(set_attr "type" "mve_move")
7494    (set_attr "length""8")])
7495
7496 ;;
7497 ;; [vsubq_m_f])
7498 ;;
7499 (define_insn "mve_vsubq_m_f<mode>"
7500   [
7501    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7502         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7503                        (match_operand:MVE_0 2 "s_register_operand" "w")
7504                        (match_operand:MVE_0 3 "s_register_operand" "w")
7505                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7506          VSUBQ_M_F))
7507   ]
7508   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7509   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7510   [(set_attr "type" "mve_move")
7511    (set_attr "length""8")])
7512
7513 ;;
7514 ;; [vsubq_m_n_f])
7515 ;;
7516 (define_insn "mve_vsubq_m_n_f<mode>"
7517   [
7518    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7519         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7520                        (match_operand:MVE_0 2 "s_register_operand" "w")
7521                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7522                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7523          VSUBQ_M_N_F))
7524   ]
7525   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7526   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7527   [(set_attr "type" "mve_move")
7528    (set_attr "length""8")])
7529
7530 ;;
7531 ;; [vstrbq_s vstrbq_u]
7532 ;;
7533 (define_insn "mve_vstrbq_<supf><mode>"
7534   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7535         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7536          VSTRBQ))
7537   ]
7538   "TARGET_HAVE_MVE"
7539 {
7540    rtx ops[2];
7541    int regno = REGNO (operands[1]);
7542    ops[1] = gen_rtx_REG (TImode, regno);
7543    ops[0]  = operands[0];
7544    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7545    return "";
7546 }
7547   [(set_attr "length" "4")])
7548
7549 ;;
7550 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7551 ;;
7552 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7553   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7554    (match_operand:MVE_2 1 "s_register_operand")
7555    (match_operand:MVE_2 2 "s_register_operand")
7556    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7557   "TARGET_HAVE_MVE"
7558 {
7559   rtx ind = XEXP (operands[0], 0);
7560   gcc_assert (REG_P (ind));
7561   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7562                                                               operands[2]));
7563   DONE;
7564 })
7565
7566 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7567   [(set (mem:BLK (scratch))
7568         (unspec:BLK
7569           [(match_operand:SI 0 "register_operand" "r")
7570            (match_operand:MVE_2 1 "s_register_operand" "w")
7571            (match_operand:MVE_2 2 "s_register_operand" "w")]
7572           VSTRBSOQ))]
7573   "TARGET_HAVE_MVE"
7574   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7575   [(set_attr "length" "4")])
7576
7577 ;;
7578 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7579 ;;
7580 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7581   [(set (mem:BLK (scratch))
7582         (unspec:BLK
7583                 [(match_operand:V4SI 0 "s_register_operand" "w")
7584                  (match_operand:SI 1 "immediate_operand" "i")
7585                  (match_operand:V4SI 2 "s_register_operand" "w")]
7586          VSTRWSBQ))
7587   ]
7588   "TARGET_HAVE_MVE"
7589 {
7590    rtx ops[3];
7591    ops[0] = operands[0];
7592    ops[1] = operands[1];
7593    ops[2] = operands[2];
7594    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7595    return "";
7596 }
7597   [(set_attr "length" "4")])
7598
7599 ;;
7600 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7601 ;;
7602 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7603   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7604         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7605                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7606          VLDRBGOQ))
7607   ]
7608   "TARGET_HAVE_MVE"
7609 {
7610    rtx ops[3];
7611    ops[0] = operands[0];
7612    ops[1] = operands[1];
7613    ops[2] = operands[2];
7614    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7615      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7616    else
7617      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7618    return "";
7619 }
7620   [(set_attr "length" "4")])
7621
7622 ;;
7623 ;; [vldrbq_s vldrbq_u]
7624 ;;
7625 (define_insn "mve_vldrbq_<supf><mode>"
7626   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7627         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7628          VLDRBQ))
7629   ]
7630   "TARGET_HAVE_MVE"
7631 {
7632    rtx ops[2];
7633    int regno = REGNO (operands[0]);
7634    ops[0] = gen_rtx_REG (TImode, regno);
7635    ops[1]  = operands[1];
7636    if (<V_sz_elem> == 8)
7637      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7638    else
7639      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7640    return "";
7641 }
7642   [(set_attr "length" "4")])
7643
7644 ;;
7645 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7646 ;;
7647 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7648   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7649         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7650                       (match_operand:SI 2 "immediate_operand" "i")]
7651          VLDRWGBQ))
7652   ]
7653   "TARGET_HAVE_MVE"
7654 {
7655    rtx ops[3];
7656    ops[0] = operands[0];
7657    ops[1] = operands[1];
7658    ops[2] = operands[2];
7659    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7660    return "";
7661 }
7662   [(set_attr "length" "4")])
7663
7664 ;;
7665 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7666 ;;
7667 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7668   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7669    (match_operand:MVE_2 1 "s_register_operand")
7670    (match_operand:MVE_2 2 "s_register_operand")
7671    (match_operand:HI 3 "vpr_register_operand" "Up")
7672    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7673   "TARGET_HAVE_MVE"
7674 {
7675   rtx ind = XEXP (operands[0], 0);
7676   gcc_assert (REG_P (ind));
7677   emit_insn (
7678     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7679                                                        operands[2],
7680                                                        operands[3]));
7681   DONE;
7682 })
7683
7684 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7685   [(set (mem:BLK (scratch))
7686         (unspec:BLK
7687           [(match_operand:SI 0 "register_operand" "r")
7688            (match_operand:MVE_2 1 "s_register_operand" "w")
7689            (match_operand:MVE_2 2 "s_register_operand" "w")
7690            (match_operand:HI 3 "vpr_register_operand" "Up")]
7691           VSTRBSOQ))]
7692   "TARGET_HAVE_MVE"
7693   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7694   [(set_attr "length" "8")])
7695
7696 ;;
7697 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7698 ;;
7699 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7700   [(set (mem:BLK (scratch))
7701         (unspec:BLK
7702                 [(match_operand:V4SI 0 "s_register_operand" "w")
7703                  (match_operand:SI 1 "immediate_operand" "i")
7704                  (match_operand:V4SI 2 "s_register_operand" "w")
7705                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7706          VSTRWSBQ))
7707   ]
7708   "TARGET_HAVE_MVE"
7709 {
7710    rtx ops[3];
7711    ops[0] = operands[0];
7712    ops[1] = operands[1];
7713    ops[2] = operands[2];
7714    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7715    return "";
7716 }
7717   [(set_attr "length" "8")])
7718
7719 ;;
7720 ;; [vstrbq_p_s vstrbq_p_u]
7721 ;;
7722 (define_insn "mve_vstrbq_p_<supf><mode>"
7723   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7724         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7725                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7726          VSTRBQ))
7727   ]
7728   "TARGET_HAVE_MVE"
7729 {
7730    rtx ops[2];
7731    int regno = REGNO (operands[1]);
7732    ops[1] = gen_rtx_REG (TImode, regno);
7733    ops[0]  = operands[0];
7734    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7735    return "";
7736 }
7737   [(set_attr "length" "8")])
7738
7739 ;;
7740 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7741 ;;
7742 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7743   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7744         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7745                        (match_operand:MVE_2 2 "s_register_operand" "w")
7746                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7747          VLDRBGOQ))
7748   ]
7749   "TARGET_HAVE_MVE"
7750 {
7751    rtx ops[4];
7752    ops[0] = operands[0];
7753    ops[1] = operands[1];
7754    ops[2] = operands[2];
7755    ops[3] = operands[3];
7756    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7757      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7758    else
7759      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7760    return "";
7761 }
7762   [(set_attr "length" "8")])
7763
7764 ;;
7765 ;; [vldrbq_z_s vldrbq_z_u]
7766 ;;
7767 (define_insn "mve_vldrbq_z_<supf><mode>"
7768   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7769         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7770                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7771          VLDRBQ))
7772   ]
7773   "TARGET_HAVE_MVE"
7774 {
7775    rtx ops[2];
7776    int regno = REGNO (operands[0]);
7777    ops[0] = gen_rtx_REG (TImode, regno);
7778    ops[1]  = operands[1];
7779    if (<V_sz_elem> == 8)
7780      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7781    else
7782      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7783    return "";
7784 }
7785   [(set_attr "length" "8")])
7786
7787 ;;
7788 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7789 ;;
7790 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7791   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7792         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7793                       (match_operand:SI 2 "immediate_operand" "i")
7794                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7795          VLDRWGBQ))
7796   ]
7797   "TARGET_HAVE_MVE"
7798 {
7799    rtx ops[3];
7800    ops[0] = operands[0];
7801    ops[1] = operands[1];
7802    ops[2] = operands[2];
7803    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7804    return "";
7805 }
7806   [(set_attr "length" "8")])
7807
7808 ;;
7809 ;; [vldrhq_f]
7810 ;;
7811 (define_insn "mve_vldrhq_fv8hf"
7812   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7813         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7814          VLDRHQ_F))
7815   ]
7816   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7817 {
7818    rtx ops[2];
7819    int regno = REGNO (operands[0]);
7820    ops[0] = gen_rtx_REG (TImode, regno);
7821    ops[1]  = operands[1];
7822    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7823    return "";
7824 }
7825   [(set_attr "length" "4")])
7826
7827 ;;
7828 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7829 ;;
7830 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7831   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7832         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7833                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7834         VLDRHGOQ))
7835   ]
7836   "TARGET_HAVE_MVE"
7837 {
7838    rtx ops[3];
7839    ops[0] = operands[0];
7840    ops[1] = operands[1];
7841    ops[2] = operands[2];
7842    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7843      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7844    else
7845      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7846    return "";
7847 }
7848   [(set_attr "length" "4")])
7849
7850 ;;
7851 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7852 ;;
7853 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7854   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7855         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7856                        (match_operand:MVE_6 2 "s_register_operand" "w")
7857                        (match_operand:HI 3 "vpr_register_operand" "Up")
7858         ]VLDRHGOQ))
7859   ]
7860   "TARGET_HAVE_MVE"
7861 {
7862    rtx ops[4];
7863    ops[0] = operands[0];
7864    ops[1] = operands[1];
7865    ops[2] = operands[2];
7866    ops[3] = operands[3];
7867    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7868      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7869    else
7870      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7871    return "";
7872 }
7873  [(set_attr "length" "8")])
7874
7875 ;;
7876 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7877 ;;
7878 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7879   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7880         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7881                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7882         VLDRHGSOQ))
7883   ]
7884   "TARGET_HAVE_MVE"
7885 {
7886    rtx ops[3];
7887    ops[0] = operands[0];
7888    ops[1] = operands[1];
7889    ops[2] = operands[2];
7890       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7891      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7892    else
7893      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7894    return "";
7895 }
7896   [(set_attr "length" "4")])
7897
7898 ;;
7899 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7900 ;;
7901 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7902   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7903         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7904                        (match_operand:MVE_6 2 "s_register_operand" "w")
7905                        (match_operand:HI 3 "vpr_register_operand" "Up")
7906         ]VLDRHGSOQ))
7907   ]
7908   "TARGET_HAVE_MVE"
7909 {
7910    rtx ops[4];
7911    ops[0] = operands[0];
7912    ops[1] = operands[1];
7913    ops[2] = operands[2];
7914    ops[3] = operands[3];
7915    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7916      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7917    else
7918      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7919    return "";
7920 }
7921   [(set_attr "length" "8")])
7922
7923 ;;
7924 ;; [vldrhq_s, vldrhq_u]
7925 ;;
7926 (define_insn "mve_vldrhq_<supf><mode>"
7927   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7928         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7929          VLDRHQ))
7930   ]
7931   "TARGET_HAVE_MVE"
7932 {
7933    rtx ops[2];
7934    int regno = REGNO (operands[0]);
7935    ops[0] = gen_rtx_REG (TImode, regno);
7936    ops[1]  = operands[1];
7937    if (<V_sz_elem> == 16)
7938      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7939    else
7940      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7941    return "";
7942 }
7943   [(set_attr "length" "4")])
7944
7945 ;;
7946 ;; [vldrhq_z_f]
7947 ;;
7948 (define_insn "mve_vldrhq_z_fv8hf"
7949   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7950         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7951         (match_operand:HI 2 "vpr_register_operand" "Up")]
7952          VLDRHQ_F))
7953   ]
7954   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7955 {
7956    rtx ops[2];
7957    int regno = REGNO (operands[0]);
7958    ops[0] = gen_rtx_REG (TImode, regno);
7959    ops[1]  = operands[1];
7960    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7961    return "";
7962 }
7963   [(set_attr "length" "8")])
7964
7965 ;;
7966 ;; [vldrhq_z_s vldrhq_z_u]
7967 ;;
7968 (define_insn "mve_vldrhq_z_<supf><mode>"
7969   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7970         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7971         (match_operand:HI 2 "vpr_register_operand" "Up")]
7972          VLDRHQ))
7973   ]
7974   "TARGET_HAVE_MVE"
7975 {
7976    rtx ops[2];
7977    int regno = REGNO (operands[0]);
7978    ops[0] = gen_rtx_REG (TImode, regno);
7979    ops[1]  = operands[1];
7980    if (<V_sz_elem> == 16)
7981      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7982    else
7983      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7984    return "";
7985 }
7986   [(set_attr "length" "8")])
7987
7988 ;;
7989 ;; [vldrwq_f]
7990 ;;
7991 (define_insn "mve_vldrwq_fv4sf"
7992   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7993         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7994          VLDRWQ_F))
7995   ]
7996   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7997 {
7998    rtx ops[2];
7999    int regno = REGNO (operands[0]);
8000    ops[0] = gen_rtx_REG (TImode, regno);
8001    ops[1]  = operands[1];
8002    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8003    return "";
8004 }
8005   [(set_attr "length" "4")])
8006
8007 ;;
8008 ;; [vldrwq_s vldrwq_u]
8009 ;;
8010 (define_insn "mve_vldrwq_<supf>v4si"
8011   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8012         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
8013          VLDRWQ))
8014   ]
8015   "TARGET_HAVE_MVE"
8016 {
8017    rtx ops[2];
8018    int regno = REGNO (operands[0]);
8019    ops[0] = gen_rtx_REG (TImode, regno);
8020    ops[1]  = operands[1];
8021    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
8022    return "";
8023 }
8024   [(set_attr "length" "4")])
8025
8026 ;;
8027 ;; [vldrwq_z_f]
8028 ;;
8029 (define_insn "mve_vldrwq_z_fv4sf"
8030   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8031         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
8032         (match_operand:HI 2 "vpr_register_operand" "Up")]
8033          VLDRWQ_F))
8034   ]
8035   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8036 {
8037    rtx ops[2];
8038    int regno = REGNO (operands[0]);
8039    ops[0] = gen_rtx_REG (TImode, regno);
8040    ops[1]  = operands[1];
8041    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8042    return "";
8043 }
8044   [(set_attr "length" "8")])
8045
8046 ;;
8047 ;; [vldrwq_z_s vldrwq_z_u]
8048 ;;
8049 (define_insn "mve_vldrwq_z_<supf>v4si"
8050   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8051         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
8052         (match_operand:HI 2 "vpr_register_operand" "Up")]
8053          VLDRWQ))
8054   ]
8055   "TARGET_HAVE_MVE"
8056 {
8057    rtx ops[2];
8058    int regno = REGNO (operands[0]);
8059    ops[0] = gen_rtx_REG (TImode, regno);
8060    ops[1]  = operands[1];
8061    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
8062    return "";
8063 }
8064   [(set_attr "length" "8")])
8065
8066 (define_expand "mve_vld1q_f<mode>"
8067   [(match_operand:MVE_0 0 "s_register_operand")
8068    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
8069   ]
8070   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8071 {
8072   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8073   DONE;
8074 })
8075
8076 (define_expand "mve_vld1q_<supf><mode>"
8077   [(match_operand:MVE_2 0 "s_register_operand")
8078    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
8079   ]
8080   "TARGET_HAVE_MVE"
8081 {
8082   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8083   DONE;
8084 })
8085
8086 ;;
8087 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8088 ;;
8089 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8090   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8091         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8092                       (match_operand:SI 2 "immediate_operand" "i")]
8093          VLDRDGBQ))
8094   ]
8095   "TARGET_HAVE_MVE"
8096 {
8097    rtx ops[3];
8098    ops[0] = operands[0];
8099    ops[1] = operands[1];
8100    ops[2] = operands[2];
8101    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8102    return "";
8103 }
8104   [(set_attr "length" "4")])
8105
8106 ;;
8107 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8108 ;;
8109 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8110   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8111         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8112                       (match_operand:SI 2 "immediate_operand" "i")
8113                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8114          VLDRDGBQ))
8115   ]
8116   "TARGET_HAVE_MVE"
8117 {
8118    rtx ops[3];
8119    ops[0] = operands[0];
8120    ops[1] = operands[1];
8121    ops[2] = operands[2];
8122    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8123    return "";
8124 }
8125   [(set_attr "length" "8")])
8126
8127 ;;
8128 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8129 ;;
8130 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8131  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8132        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8133                      (match_operand:V2DI 2 "s_register_operand" "w")]
8134         VLDRDGOQ))
8135  ]
8136  "TARGET_HAVE_MVE"
8137 {
8138   rtx ops[3];
8139   ops[0] = operands[0];
8140   ops[1] = operands[1];
8141   ops[2] = operands[2];
8142   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8143   return "";
8144 }
8145  [(set_attr "length" "4")])
8146
8147 ;;
8148 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8149 ;;
8150 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8151  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8152        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8153                      (match_operand:V2DI 2 "s_register_operand" "w")
8154                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8155         VLDRDGOQ))
8156  ]
8157  "TARGET_HAVE_MVE"
8158 {
8159   rtx ops[3];
8160   ops[0] = operands[0];
8161   ops[1] = operands[1];
8162   ops[2] = operands[2];
8163   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8164   return "";
8165 }
8166  [(set_attr "length" "8")])
8167
8168 ;;
8169 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8170 ;;
8171 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8172   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8173         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8174                       (match_operand:V2DI 2 "s_register_operand" "w")]
8175          VLDRDGSOQ))
8176   ]
8177   "TARGET_HAVE_MVE"
8178 {
8179    rtx ops[3];
8180    ops[0] = operands[0];
8181    ops[1] = operands[1];
8182    ops[2] = operands[2];
8183    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8184    return "";
8185 }
8186   [(set_attr "length" "4")])
8187
8188 ;;
8189 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8190 ;;
8191 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8192   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8193         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8194                       (match_operand:V2DI 2 "s_register_operand" "w")
8195                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8196          VLDRDGSOQ))
8197   ]
8198   "TARGET_HAVE_MVE"
8199 {
8200    rtx ops[3];
8201    ops[0] = operands[0];
8202    ops[1] = operands[1];
8203    ops[2] = operands[2];
8204    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8205    return "";
8206 }
8207   [(set_attr "length" "8")])
8208
8209 ;;
8210 ;; [vldrhq_gather_offset_f]
8211 ;;
8212 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8213   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8214         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8215                       (match_operand:V8HI 2 "s_register_operand" "w")]
8216          VLDRHQGO_F))
8217   ]
8218   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8219 {
8220    rtx ops[3];
8221    ops[0] = operands[0];
8222    ops[1] = operands[1];
8223    ops[2] = operands[2];
8224    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8225    return "";
8226 }
8227   [(set_attr "length" "4")])
8228
8229 ;;
8230 ;; [vldrhq_gather_offset_z_f]
8231 ;;
8232 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8233   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8234         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8235                       (match_operand:V8HI 2 "s_register_operand" "w")
8236                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8237          VLDRHQGO_F))
8238   ]
8239   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8240 {
8241    rtx ops[4];
8242    ops[0] = operands[0];
8243    ops[1] = operands[1];
8244    ops[2] = operands[2];
8245    ops[3] = operands[3];
8246    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8247    return "";
8248 }
8249   [(set_attr "length" "8")])
8250
8251 ;;
8252 ;; [vldrhq_gather_shifted_offset_f]
8253 ;;
8254 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8255   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8256         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8257                       (match_operand:V8HI 2 "s_register_operand" "w")]
8258          VLDRHQGSO_F))
8259   ]
8260   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8261 {
8262    rtx ops[3];
8263    ops[0] = operands[0];
8264    ops[1] = operands[1];
8265    ops[2] = operands[2];
8266    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8267    return "";
8268 }
8269   [(set_attr "length" "4")])
8270
8271 ;;
8272 ;; [vldrhq_gather_shifted_offset_z_f]
8273 ;;
8274 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8275   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8276         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8277                       (match_operand:V8HI 2 "s_register_operand" "w")
8278                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8279          VLDRHQGSO_F))
8280   ]
8281   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8282 {
8283    rtx ops[4];
8284    ops[0] = operands[0];
8285    ops[1] = operands[1];
8286    ops[2] = operands[2];
8287    ops[3] = operands[3];
8288    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8289    return "";
8290 }
8291   [(set_attr "length" "8")])
8292
8293 ;;
8294 ;; [vldrwq_gather_base_f]
8295 ;;
8296 (define_insn "mve_vldrwq_gather_base_fv4sf"
8297   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8298         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8299                       (match_operand:SI 2 "immediate_operand" "i")]
8300          VLDRWQGB_F))
8301   ]
8302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8303 {
8304    rtx ops[3];
8305    ops[0] = operands[0];
8306    ops[1] = operands[1];
8307    ops[2] = operands[2];
8308    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8309    return "";
8310 }
8311   [(set_attr "length" "4")])
8312
8313 ;;
8314 ;; [vldrwq_gather_base_z_f]
8315 ;;
8316 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8317   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8318         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8319                       (match_operand:SI 2 "immediate_operand" "i")
8320                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8321          VLDRWQGB_F))
8322   ]
8323   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8324 {
8325    rtx ops[3];
8326    ops[0] = operands[0];
8327    ops[1] = operands[1];
8328    ops[2] = operands[2];
8329    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8330    return "";
8331 }
8332   [(set_attr "length" "8")])
8333
8334 ;;
8335 ;; [vldrwq_gather_offset_f]
8336 ;;
8337 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8338   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8339         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8340                        (match_operand:V4SI 2 "s_register_operand" "w")]
8341          VLDRWQGO_F))
8342   ]
8343   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8344 {
8345    rtx ops[3];
8346    ops[0] = operands[0];
8347    ops[1] = operands[1];
8348    ops[2] = operands[2];
8349    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8350    return "";
8351 }
8352   [(set_attr "length" "4")])
8353
8354 ;;
8355 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8356 ;;
8357 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8358   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8359         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8360                        (match_operand:V4SI 2 "s_register_operand" "w")]
8361          VLDRWGOQ))
8362   ]
8363   "TARGET_HAVE_MVE"
8364 {
8365    rtx ops[3];
8366    ops[0] = operands[0];
8367    ops[1] = operands[1];
8368    ops[2] = operands[2];
8369    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8370    return "";
8371 }
8372   [(set_attr "length" "4")])
8373
8374 ;;
8375 ;; [vldrwq_gather_offset_z_f]
8376 ;;
8377 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8378   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8379         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8380                       (match_operand:V4SI 2 "s_register_operand" "w")
8381                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8382          VLDRWQGO_F))
8383   ]
8384   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8385 {
8386    rtx ops[4];
8387    ops[0] = operands[0];
8388    ops[1] = operands[1];
8389    ops[2] = operands[2];
8390    ops[3] = operands[3];
8391    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8392    return "";
8393 }
8394   [(set_attr "length" "8")])
8395
8396 ;;
8397 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8398 ;;
8399 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8400   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8401         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8402                       (match_operand:V4SI 2 "s_register_operand" "w")
8403                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8404          VLDRWGOQ))
8405   ]
8406   "TARGET_HAVE_MVE"
8407 {
8408    rtx ops[4];
8409    ops[0] = operands[0];
8410    ops[1] = operands[1];
8411    ops[2] = operands[2];
8412    ops[3] = operands[3];
8413    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8414    return "";
8415 }
8416   [(set_attr "length" "8")])
8417
8418 ;;
8419 ;; [vldrwq_gather_shifted_offset_f]
8420 ;;
8421 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8422   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8423         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8424                       (match_operand:V4SI 2 "s_register_operand" "w")]
8425          VLDRWQGSO_F))
8426   ]
8427   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8428 {
8429    rtx ops[3];
8430    ops[0] = operands[0];
8431    ops[1] = operands[1];
8432    ops[2] = operands[2];
8433    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8434    return "";
8435 }
8436   [(set_attr "length" "4")])
8437
8438 ;;
8439 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8440 ;;
8441 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8442   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8443         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8444                       (match_operand:V4SI 2 "s_register_operand" "w")]
8445          VLDRWGSOQ))
8446   ]
8447   "TARGET_HAVE_MVE"
8448 {
8449    rtx ops[3];
8450    ops[0] = operands[0];
8451    ops[1] = operands[1];
8452    ops[2] = operands[2];
8453    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8454    return "";
8455 }
8456   [(set_attr "length" "4")])
8457
8458 ;;
8459 ;; [vldrwq_gather_shifted_offset_z_f]
8460 ;;
8461 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8462   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8463         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8464                       (match_operand:V4SI 2 "s_register_operand" "w")
8465                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8466          VLDRWQGSO_F))
8467   ]
8468   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8469 {
8470    rtx ops[4];
8471    ops[0] = operands[0];
8472    ops[1] = operands[1];
8473    ops[2] = operands[2];
8474    ops[3] = operands[3];
8475    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8476    return "";
8477 }
8478   [(set_attr "length" "8")])
8479
8480 ;;
8481 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8482 ;;
8483 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8484   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8485         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8486                       (match_operand:V4SI 2 "s_register_operand" "w")
8487                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8488          VLDRWGSOQ))
8489   ]
8490   "TARGET_HAVE_MVE"
8491 {
8492    rtx ops[4];
8493    ops[0] = operands[0];
8494    ops[1] = operands[1];
8495    ops[2] = operands[2];
8496    ops[3] = operands[3];
8497    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8498    return "";
8499 }
8500   [(set_attr "length" "8")])
8501
8502 ;;
8503 ;; [vstrhq_f]
8504 ;;
8505 (define_insn "mve_vstrhq_fv8hf"
8506   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8507         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8508          VSTRHQ_F))
8509   ]
8510   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8511 {
8512    rtx ops[2];
8513    int regno = REGNO (operands[1]);
8514    ops[1] = gen_rtx_REG (TImode, regno);
8515    ops[0]  = operands[0];
8516    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8517    return "";
8518 }
8519   [(set_attr "length" "4")])
8520
8521 ;;
8522 ;; [vstrhq_p_f]
8523 ;;
8524 (define_insn "mve_vstrhq_p_fv8hf"
8525   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8526         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8527                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8528          VSTRHQ_F))
8529   ]
8530   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8531 {
8532    rtx ops[2];
8533    int regno = REGNO (operands[1]);
8534    ops[1] = gen_rtx_REG (TImode, regno);
8535    ops[0]  = operands[0];
8536    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8537    return "";
8538 }
8539   [(set_attr "length" "8")])
8540
8541 ;;
8542 ;; [vstrhq_p_s vstrhq_p_u]
8543 ;;
8544 (define_insn "mve_vstrhq_p_<supf><mode>"
8545   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8546         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8547                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8548          VSTRHQ))
8549   ]
8550   "TARGET_HAVE_MVE"
8551 {
8552    rtx ops[2];
8553    int regno = REGNO (operands[1]);
8554    ops[1] = gen_rtx_REG (TImode, regno);
8555    ops[0]  = operands[0];
8556    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8557    return "";
8558 }
8559   [(set_attr "length" "8")])
8560
8561 ;;
8562 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8563 ;;
8564 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8565   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8566    (match_operand:MVE_6 1 "s_register_operand")
8567    (match_operand:MVE_6 2 "s_register_operand")
8568    (match_operand:HI 3 "vpr_register_operand")
8569    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8570   "TARGET_HAVE_MVE"
8571 {
8572   rtx ind = XEXP (operands[0], 0);
8573   gcc_assert (REG_P (ind));
8574   emit_insn (
8575     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8576                                                        operands[2],
8577                                                        operands[3]));
8578   DONE;
8579 })
8580
8581 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8582   [(set (mem:BLK (scratch))
8583         (unspec:BLK
8584           [(match_operand:SI 0 "register_operand" "r")
8585            (match_operand:MVE_6 1 "s_register_operand" "w")
8586            (match_operand:MVE_6 2 "s_register_operand" "w")
8587            (match_operand:HI 3 "vpr_register_operand" "Up")]
8588           VSTRHSOQ))]
8589   "TARGET_HAVE_MVE"
8590   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8591   [(set_attr "length" "8")])
8592
8593 ;;
8594 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8595 ;;
8596 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8597   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8598    (match_operand:MVE_6 1 "s_register_operand")
8599    (match_operand:MVE_6 2 "s_register_operand")
8600    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8601   "TARGET_HAVE_MVE"
8602 {
8603   rtx ind = XEXP (operands[0], 0);
8604   gcc_assert (REG_P (ind));
8605   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8606                                                               operands[2]));
8607   DONE;
8608 })
8609
8610 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8611   [(set (mem:BLK (scratch))
8612         (unspec:BLK
8613           [(match_operand:SI 0 "register_operand" "r")
8614            (match_operand:MVE_6 1 "s_register_operand" "w")
8615            (match_operand:MVE_6 2 "s_register_operand" "w")]
8616           VSTRHSOQ))]
8617   "TARGET_HAVE_MVE"
8618   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8619   [(set_attr "length" "4")])
8620
8621 ;;
8622 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8623 ;;
8624 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8625   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8626    (match_operand:MVE_6 1 "s_register_operand")
8627    (match_operand:MVE_6 2 "s_register_operand")
8628    (match_operand:HI 3 "vpr_register_operand")
8629    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8630   "TARGET_HAVE_MVE"
8631 {
8632   rtx ind = XEXP (operands[0], 0);
8633   gcc_assert (REG_P (ind));
8634   emit_insn (
8635     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8636                                                                operands[2],
8637                                                                operands[3]));
8638   DONE;
8639 })
8640
8641 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8642   [(set (mem:BLK (scratch))
8643         (unspec:BLK
8644           [(match_operand:SI 0 "register_operand" "r")
8645            (match_operand:MVE_6 1 "s_register_operand" "w")
8646            (match_operand:MVE_6 2 "s_register_operand" "w")
8647            (match_operand:HI 3 "vpr_register_operand" "Up")]
8648           VSTRHSSOQ))]
8649   "TARGET_HAVE_MVE"
8650   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8651   [(set_attr "length" "8")])
8652
8653 ;;
8654 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8655 ;;
8656 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8657   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8658    (match_operand:MVE_6 1 "s_register_operand")
8659    (match_operand:MVE_6 2 "s_register_operand")
8660    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8661   "TARGET_HAVE_MVE"
8662 {
8663   rtx ind = XEXP (operands[0], 0);
8664   gcc_assert (REG_P (ind));
8665   emit_insn (
8666     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8667                                                              operands[2]));
8668   DONE;
8669 })
8670
8671 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8672   [(set (mem:BLK (scratch))
8673         (unspec:BLK
8674           [(match_operand:SI 0 "register_operand" "r")
8675            (match_operand:MVE_6 1 "s_register_operand" "w")
8676            (match_operand:MVE_6 2 "s_register_operand" "w")]
8677           VSTRHSSOQ))]
8678   "TARGET_HAVE_MVE"
8679   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8680   [(set_attr "length" "4")])
8681
8682 ;;
8683 ;; [vstrhq_s, vstrhq_u]
8684 ;;
8685 (define_insn "mve_vstrhq_<supf><mode>"
8686   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8687         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8688          VSTRHQ))
8689   ]
8690   "TARGET_HAVE_MVE"
8691 {
8692    rtx ops[2];
8693    int regno = REGNO (operands[1]);
8694    ops[1] = gen_rtx_REG (TImode, regno);
8695    ops[0]  = operands[0];
8696    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8697    return "";
8698 }
8699   [(set_attr "length" "4")])
8700
8701 ;;
8702 ;; [vstrwq_f]
8703 ;;
8704 (define_insn "mve_vstrwq_fv4sf"
8705   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8706         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8707          VSTRWQ_F))
8708   ]
8709   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8710 {
8711    rtx ops[2];
8712    int regno = REGNO (operands[1]);
8713    ops[1] = gen_rtx_REG (TImode, regno);
8714    ops[0]  = operands[0];
8715    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8716    return "";
8717 }
8718   [(set_attr "length" "4")])
8719
8720 ;;
8721 ;; [vstrwq_p_f]
8722 ;;
8723 (define_insn "mve_vstrwq_p_fv4sf"
8724   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8725         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8726                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8727          VSTRWQ_F))
8728   ]
8729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8730 {
8731    rtx ops[2];
8732    int regno = REGNO (operands[1]);
8733    ops[1] = gen_rtx_REG (TImode, regno);
8734    ops[0]  = operands[0];
8735    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8736    return "";
8737 }
8738   [(set_attr "length" "8")])
8739
8740 ;;
8741 ;; [vstrwq_p_s vstrwq_p_u]
8742 ;;
8743 (define_insn "mve_vstrwq_p_<supf>v4si"
8744   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8745         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8746                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8747          VSTRWQ))
8748   ]
8749   "TARGET_HAVE_MVE"
8750 {
8751    rtx ops[2];
8752    int regno = REGNO (operands[1]);
8753    ops[1] = gen_rtx_REG (TImode, regno);
8754    ops[0]  = operands[0];
8755    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8756    return "";
8757 }
8758   [(set_attr "length" "8")])
8759
8760 ;;
8761 ;; [vstrwq_s vstrwq_u]
8762 ;;
8763 (define_insn "mve_vstrwq_<supf>v4si"
8764   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8765         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8766          VSTRWQ))
8767   ]
8768   "TARGET_HAVE_MVE"
8769 {
8770    rtx ops[2];
8771    int regno = REGNO (operands[1]);
8772    ops[1] = gen_rtx_REG (TImode, regno);
8773    ops[0]  = operands[0];
8774    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8775    return "";
8776 }
8777   [(set_attr "length" "4")])
8778
8779 (define_expand "mve_vst1q_f<mode>"
8780   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8781    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8782   ]
8783   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8784 {
8785   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8786   DONE;
8787 })
8788
8789 (define_expand "mve_vst1q_<supf><mode>"
8790   [(match_operand:MVE_2 0 "mve_memory_operand")
8791    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8792   ]
8793   "TARGET_HAVE_MVE"
8794 {
8795   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8796   DONE;
8797 })
8798
8799 ;;
8800 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8801 ;;
8802 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8803   [(set (mem:BLK (scratch))
8804         (unspec:BLK
8805                 [(match_operand:V2DI 0 "s_register_operand" "w")
8806                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8807                  (match_operand:V2DI 2 "s_register_operand" "w")
8808                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8809          VSTRDSBQ))
8810   ]
8811   "TARGET_HAVE_MVE"
8812 {
8813    rtx ops[3];
8814    ops[0] = operands[0];
8815    ops[1] = operands[1];
8816    ops[2] = operands[2];
8817    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8818    return "";
8819 }
8820   [(set_attr "length" "8")])
8821
8822 ;;
8823 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8824 ;;
8825 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8826   [(set (mem:BLK (scratch))
8827         (unspec:BLK
8828                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8829                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8830                  (match_operand:V2DI 2 "s_register_operand" "w")]
8831          VSTRDSBQ))
8832   ]
8833   "TARGET_HAVE_MVE"
8834 {
8835    rtx ops[3];
8836    ops[0] = operands[0];
8837    ops[1] = operands[1];
8838    ops[2] = operands[2];
8839    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8840    return "";
8841 }
8842   [(set_attr "length" "4")])
8843
8844 ;;
8845 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8846 ;;
8847 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8848   [(match_operand:V2DI 0 "mve_scatter_memory")
8849    (match_operand:V2DI 1 "s_register_operand")
8850    (match_operand:V2DI 2 "s_register_operand")
8851    (match_operand:HI 3 "vpr_register_operand")
8852    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8853   "TARGET_HAVE_MVE"
8854 {
8855   rtx ind = XEXP (operands[0], 0);
8856   gcc_assert (REG_P (ind));
8857   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8858                                                               operands[2],
8859                                                               operands[3]));
8860   DONE;
8861 })
8862
8863 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8864   [(set (mem:BLK (scratch))
8865         (unspec:BLK
8866           [(match_operand:SI 0 "register_operand" "r")
8867            (match_operand:V2DI 1 "s_register_operand" "w")
8868            (match_operand:V2DI 2 "s_register_operand" "w")
8869            (match_operand:HI 3 "vpr_register_operand" "Up")]
8870           VSTRDSOQ))]
8871   "TARGET_HAVE_MVE"
8872   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8873   [(set_attr "length" "8")])
8874
8875 ;;
8876 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8877 ;;
8878 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8879   [(match_operand:V2DI 0 "mve_scatter_memory")
8880    (match_operand:V2DI 1 "s_register_operand")
8881    (match_operand:V2DI 2 "s_register_operand")
8882    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8883   "TARGET_HAVE_MVE"
8884 {
8885   rtx ind = XEXP (operands[0], 0);
8886   gcc_assert (REG_P (ind));
8887   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8888                                                             operands[2]));
8889   DONE;
8890 })
8891
8892 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8893   [(set (mem:BLK (scratch))
8894         (unspec:BLK
8895           [(match_operand:SI 0 "register_operand" "r")
8896            (match_operand:V2DI 1 "s_register_operand" "w")
8897            (match_operand:V2DI 2 "s_register_operand" "w")]
8898           VSTRDSOQ))]
8899   "TARGET_HAVE_MVE"
8900   "vstrd.64\t%q2, [%0, %q1]"
8901   [(set_attr "length" "4")])
8902
8903 ;;
8904 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8905 ;;
8906 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8907   [(match_operand:V2DI 0 "mve_scatter_memory")
8908    (match_operand:V2DI 1 "s_register_operand")
8909    (match_operand:V2DI 2 "s_register_operand")
8910    (match_operand:HI 3 "vpr_register_operand")
8911    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8912   "TARGET_HAVE_MVE"
8913 {
8914   rtx ind = XEXP (operands[0], 0);
8915   gcc_assert (REG_P (ind));
8916   emit_insn (
8917     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8918                                                              operands[2],
8919                                                              operands[3]));
8920   DONE;
8921 })
8922
8923 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8924   [(set (mem:BLK (scratch))
8925         (unspec:BLK
8926           [(match_operand:SI 0 "register_operand" "r")
8927            (match_operand:V2DI 1 "s_register_operand" "w")
8928            (match_operand:V2DI 2 "s_register_operand" "w")
8929            (match_operand:HI 3 "vpr_register_operand" "Up")]
8930           VSTRDSSOQ))]
8931   "TARGET_HAVE_MVE"
8932   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8933   [(set_attr "length" "8")])
8934
8935 ;;
8936 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8937 ;;
8938 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8939   [(match_operand:V2DI 0 "mve_scatter_memory")
8940    (match_operand:V2DI 1 "s_register_operand")
8941    (match_operand:V2DI 2 "s_register_operand")
8942    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8943   "TARGET_HAVE_MVE"
8944 {
8945   rtx ind = XEXP (operands[0], 0);
8946   gcc_assert (REG_P (ind));
8947   emit_insn (
8948     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8949                                                            operands[2]));
8950   DONE;
8951 })
8952
8953 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8954   [(set (mem:BLK (scratch))
8955         (unspec:BLK
8956           [(match_operand:SI 0 "register_operand" "r")
8957            (match_operand:V2DI 1 "s_register_operand" "w")
8958            (match_operand:V2DI 2 "s_register_operand" "w")]
8959           VSTRDSSOQ))]
8960   "TARGET_HAVE_MVE"
8961   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8962   [(set_attr "length" "4")])
8963
8964 ;;
8965 ;; [vstrhq_scatter_offset_f]
8966 ;;
8967 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8968   [(match_operand:V8HI 0 "mve_scatter_memory")
8969    (match_operand:V8HI 1 "s_register_operand")
8970    (match_operand:V8HF 2 "s_register_operand")
8971    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8972   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8973 {
8974   rtx ind = XEXP (operands[0], 0);
8975   gcc_assert (REG_P (ind));
8976   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8977                                                        operands[2]));
8978   DONE;
8979 })
8980
8981 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8982   [(set (mem:BLK (scratch))
8983         (unspec:BLK
8984           [(match_operand:SI 0 "register_operand" "r")
8985            (match_operand:V8HI 1 "s_register_operand" "w")
8986            (match_operand:V8HF 2 "s_register_operand" "w")]
8987           VSTRHQSO_F))]
8988   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8989   "vstrh.16\t%q2, [%0, %q1]"
8990   [(set_attr "length" "4")])
8991
8992 ;;
8993 ;; [vstrhq_scatter_offset_p_f]
8994 ;;
8995 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8996   [(match_operand:V8HI 0 "mve_scatter_memory")
8997    (match_operand:V8HI 1 "s_register_operand")
8998    (match_operand:V8HF 2 "s_register_operand")
8999    (match_operand:HI 3 "vpr_register_operand")
9000    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
9001   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9002 {
9003   rtx ind = XEXP (operands[0], 0);
9004   gcc_assert (REG_P (ind));
9005   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
9006                                                          operands[2],
9007                                                          operands[3]));
9008   DONE;
9009 })
9010
9011 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
9012   [(set (mem:BLK (scratch))
9013         (unspec:BLK
9014           [(match_operand:SI 0 "register_operand" "r")
9015            (match_operand:V8HI 1 "s_register_operand" "w")
9016            (match_operand:V8HF 2 "s_register_operand" "w")
9017            (match_operand:HI 3 "vpr_register_operand" "Up")]
9018           VSTRHQSO_F))]
9019   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9020   "vpst\;vstrht.16\t%q2, [%0, %q1]"
9021   [(set_attr "length" "8")])
9022
9023 ;;
9024 ;; [vstrhq_scatter_shifted_offset_f]
9025 ;;
9026 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
9027   [(match_operand:V8HI 0 "memory_operand" "=Us")
9028    (match_operand:V8HI 1 "s_register_operand" "w")
9029    (match_operand:V8HF 2 "s_register_operand" "w")
9030    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9031   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9032 {
9033   rtx ind = XEXP (operands[0], 0);
9034   gcc_assert (REG_P (ind));
9035   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
9036                                                                operands[2]));
9037   DONE;
9038 })
9039
9040 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
9041   [(set (mem:BLK (scratch))
9042         (unspec:BLK
9043           [(match_operand:SI 0 "register_operand" "r")
9044            (match_operand:V8HI 1 "s_register_operand" "w")
9045            (match_operand:V8HF 2 "s_register_operand" "w")]
9046           VSTRHQSSO_F))]
9047   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9048   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
9049   [(set_attr "length" "4")])
9050
9051 ;;
9052 ;; [vstrhq_scatter_shifted_offset_p_f]
9053 ;;
9054 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9055   [(match_operand:V8HI 0 "memory_operand" "=Us")
9056    (match_operand:V8HI 1 "s_register_operand" "w")
9057    (match_operand:V8HF 2 "s_register_operand" "w")
9058    (match_operand:HI 3 "vpr_register_operand" "Up")
9059    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
9060   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9061 {
9062   rtx ind = XEXP (operands[0], 0);
9063   gcc_assert (REG_P (ind));
9064   emit_insn (
9065     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
9066                                                         operands[2],
9067                                                         operands[3]));
9068   DONE;
9069 })
9070
9071 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
9072   [(set (mem:BLK (scratch))
9073         (unspec:BLK
9074           [(match_operand:SI 0 "register_operand" "r")
9075            (match_operand:V8HI 1 "s_register_operand" "w")
9076            (match_operand:V8HF 2 "s_register_operand" "w")
9077            (match_operand:HI 3 "vpr_register_operand" "Up")]
9078           VSTRHQSSO_F))]
9079   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9080   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
9081   [(set_attr "length" "8")])
9082
9083 ;;
9084 ;; [vstrwq_scatter_base_f]
9085 ;;
9086 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9087   [(set (mem:BLK (scratch))
9088         (unspec:BLK
9089                 [(match_operand:V4SI 0 "s_register_operand" "w")
9090                  (match_operand:SI 1 "immediate_operand" "i")
9091                  (match_operand:V4SF 2 "s_register_operand" "w")]
9092          VSTRWQSB_F))
9093   ]
9094   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9095 {
9096    rtx ops[3];
9097    ops[0] = operands[0];
9098    ops[1] = operands[1];
9099    ops[2] = operands[2];
9100    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9101    return "";
9102 }
9103   [(set_attr "length" "4")])
9104
9105 ;;
9106 ;; [vstrwq_scatter_base_p_f]
9107 ;;
9108 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9109   [(set (mem:BLK (scratch))
9110         (unspec:BLK
9111                 [(match_operand:V4SI 0 "s_register_operand" "w")
9112                  (match_operand:SI 1 "immediate_operand" "i")
9113                  (match_operand:V4SF 2 "s_register_operand" "w")
9114                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9115          VSTRWQSB_F))
9116   ]
9117   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9118 {
9119    rtx ops[3];
9120    ops[0] = operands[0];
9121    ops[1] = operands[1];
9122    ops[2] = operands[2];
9123    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9124    return "";
9125 }
9126   [(set_attr "length" "8")])
9127
9128 ;;
9129 ;; [vstrwq_scatter_offset_f]
9130 ;;
9131 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
9132   [(match_operand:V4SI 0 "mve_scatter_memory")
9133    (match_operand:V4SI 1 "s_register_operand")
9134    (match_operand:V4SF 2 "s_register_operand")
9135    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9137 {
9138   rtx ind = XEXP (operands[0], 0);
9139   gcc_assert (REG_P (ind));
9140   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
9141                                                        operands[2]));
9142   DONE;
9143 })
9144
9145 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
9146   [(set (mem:BLK (scratch))
9147         (unspec:BLK
9148           [(match_operand:SI 0 "register_operand" "r")
9149            (match_operand:V4SI 1 "s_register_operand" "w")
9150            (match_operand:V4SF 2 "s_register_operand" "w")]
9151           VSTRWQSO_F))]
9152   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9153   "vstrw.32\t%q2, [%0, %q1]"
9154   [(set_attr "length" "4")])
9155
9156 ;;
9157 ;; [vstrwq_scatter_offset_p_f]
9158 ;;
9159 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
9160   [(match_operand:V4SI 0 "mve_scatter_memory")
9161    (match_operand:V4SI 1 "s_register_operand")
9162    (match_operand:V4SF 2 "s_register_operand")
9163    (match_operand:HI 3 "vpr_register_operand")
9164    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
9165   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9166 {
9167   rtx ind = XEXP (operands[0], 0);
9168   gcc_assert (REG_P (ind));
9169   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
9170                                                          operands[2],
9171                                                          operands[3]));
9172   DONE;
9173 })
9174
9175 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
9176   [(set (mem:BLK (scratch))
9177         (unspec:BLK
9178           [(match_operand:SI 0 "register_operand" "r")
9179            (match_operand:V4SI 1 "s_register_operand" "w")
9180            (match_operand:V4SF 2 "s_register_operand" "w")
9181            (match_operand:HI 3 "vpr_register_operand" "Up")]
9182           VSTRWQSO_F))]
9183   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9184   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9185   [(set_attr "length" "8")])
9186
9187 ;;
9188 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9189 ;;
9190 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
9191   [(match_operand:V4SI 0 "mve_scatter_memory")
9192    (match_operand:V4SI 1 "s_register_operand")
9193    (match_operand:V4SI 2 "s_register_operand")
9194    (match_operand:HI 3 "vpr_register_operand")
9195    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9196   "TARGET_HAVE_MVE"
9197 {
9198   rtx ind = XEXP (operands[0], 0);
9199   gcc_assert (REG_P (ind));
9200   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
9201                                                               operands[2],
9202                                                               operands[3]));
9203   DONE;
9204 })
9205
9206 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
9207   [(set (mem:BLK (scratch))
9208         (unspec:BLK
9209           [(match_operand:SI 0 "register_operand" "r")
9210            (match_operand:V4SI 1 "s_register_operand" "w")
9211            (match_operand:V4SI 2 "s_register_operand" "w")
9212            (match_operand:HI 3 "vpr_register_operand" "Up")]
9213           VSTRWSOQ))]
9214   "TARGET_HAVE_MVE"
9215   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
9216   [(set_attr "length" "8")])
9217
9218 ;;
9219 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9220 ;;
9221 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
9222   [(match_operand:V4SI 0 "mve_scatter_memory")
9223    (match_operand:V4SI 1 "s_register_operand")
9224    (match_operand:V4SI 2 "s_register_operand")
9225    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
9226   "TARGET_HAVE_MVE"
9227 {
9228   rtx ind = XEXP (operands[0], 0);
9229   gcc_assert (REG_P (ind));
9230   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
9231                                                             operands[2]));
9232   DONE;
9233 })
9234
9235 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
9236   [(set (mem:BLK (scratch))
9237         (unspec:BLK
9238           [(match_operand:SI 0 "register_operand" "r")
9239            (match_operand:V4SI 1 "s_register_operand" "w")
9240            (match_operand:V4SI 2 "s_register_operand" "w")]
9241           VSTRWSOQ))]
9242   "TARGET_HAVE_MVE"
9243   "vstrw.32\t%q2, [%0, %q1]"
9244   [(set_attr "length" "4")])
9245
9246 ;;
9247 ;; [vstrwq_scatter_shifted_offset_f]
9248 ;;
9249 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
9250   [(match_operand:V4SI 0 "mve_scatter_memory")
9251    (match_operand:V4SI 1 "s_register_operand")
9252    (match_operand:V4SF 2 "s_register_operand")
9253    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9254   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9255 {
9256   rtx ind = XEXP (operands[0], 0);
9257   gcc_assert (REG_P (ind));
9258   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
9259                                                                operands[2]));
9260   DONE;
9261 })
9262
9263 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
9264   [(set (mem:BLK (scratch))
9265         (unspec:BLK
9266           [(match_operand:SI 0 "register_operand" "r")
9267            (match_operand:V4SI 1 "s_register_operand" "w")
9268            (match_operand:V4SF 2 "s_register_operand" "w")]
9269          VSTRWQSSO_F))]
9270   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9271   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9272   [(set_attr "length" "8")])
9273
9274 ;;
9275 ;; [vstrwq_scatter_shifted_offset_p_f]
9276 ;;
9277 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9278   [(match_operand:V4SI 0 "mve_scatter_memory")
9279    (match_operand:V4SI 1 "s_register_operand")
9280    (match_operand:V4SF 2 "s_register_operand")
9281    (match_operand:HI 3 "vpr_register_operand")
9282    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
9283   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9284 {
9285   rtx ind = XEXP (operands[0], 0);
9286   gcc_assert (REG_P (ind));
9287   emit_insn (
9288     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
9289                                                         operands[2],
9290                                                         operands[3]));
9291   DONE;
9292 })
9293
9294 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
9295   [(set (mem:BLK (scratch))
9296         (unspec:BLK
9297           [(match_operand:SI 0 "register_operand" "r")
9298            (match_operand:V4SI 1 "s_register_operand" "w")
9299            (match_operand:V4SF 2 "s_register_operand" "w")
9300            (match_operand:HI 3 "vpr_register_operand" "Up")]
9301           VSTRWQSSO_F))]
9302   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9303   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9304   [(set_attr "length" "8")])
9305
9306 ;;
9307 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9308 ;;
9309 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9310   [(match_operand:V4SI 0 "mve_scatter_memory")
9311    (match_operand:V4SI 1 "s_register_operand")
9312    (match_operand:V4SI 2 "s_register_operand")
9313    (match_operand:HI 3 "vpr_register_operand")
9314    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9315   "TARGET_HAVE_MVE"
9316 {
9317   rtx ind = XEXP (operands[0], 0);
9318   gcc_assert (REG_P (ind));
9319   emit_insn (
9320     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
9321                                                              operands[2],
9322                                                              operands[3]));
9323   DONE;
9324 })
9325
9326 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
9327   [(set (mem:BLK (scratch))
9328         (unspec:BLK
9329           [(match_operand:SI 0 "register_operand" "r")
9330            (match_operand:V4SI 1 "s_register_operand" "w")
9331            (match_operand:V4SI 2 "s_register_operand" "w")
9332            (match_operand:HI 3 "vpr_register_operand" "Up")]
9333           VSTRWSSOQ))]
9334   "TARGET_HAVE_MVE"
9335   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
9336   [(set_attr "length" "8")])
9337
9338 ;;
9339 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9340 ;;
9341 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9342   [(match_operand:V4SI 0 "mve_scatter_memory")
9343    (match_operand:V4SI 1 "s_register_operand")
9344    (match_operand:V4SI 2 "s_register_operand")
9345    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
9346   "TARGET_HAVE_MVE"
9347 {
9348   rtx ind = XEXP (operands[0], 0);
9349   gcc_assert (REG_P (ind));
9350   emit_insn (
9351     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
9352                                                            operands[2]));
9353   DONE;
9354 })
9355
9356 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
9357   [(set (mem:BLK (scratch))
9358         (unspec:BLK
9359           [(match_operand:SI 0 "register_operand" "r")
9360            (match_operand:V4SI 1 "s_register_operand" "w")
9361            (match_operand:V4SI 2 "s_register_operand" "w")]
9362           VSTRWSSOQ))]
9363   "TARGET_HAVE_MVE"
9364   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
9365   [(set_attr "length" "4")])
9366
9367 ;;
9368 ;; [vaddq_s, vaddq_u])
9369 ;;
9370 (define_insn "mve_vaddq<mode>"
9371   [
9372    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9373         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9374                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9375   ]
9376   "TARGET_HAVE_MVE"
9377   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9378   [(set_attr "type" "mve_move")
9379 ])
9380
9381 ;;
9382 ;; [vaddq_f])
9383 ;;
9384 (define_insn "mve_vaddq_f<mode>"
9385   [
9386    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9387         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9388                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9389   ]
9390   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9391   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9392   [(set_attr "type" "mve_move")
9393 ])
9394
9395 ;;
9396 ;; [vidupq_n_u])
9397 ;;
9398 (define_expand "mve_vidupq_n_u<mode>"
9399  [(match_operand:MVE_2 0 "s_register_operand")
9400   (match_operand:SI 1 "s_register_operand")
9401   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9402  "TARGET_HAVE_MVE"
9403 {
9404   rtx temp = gen_reg_rtx (SImode);
9405   emit_move_insn (temp, operands[1]);
9406   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9407   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9408                                           operands[2], inc));
9409   DONE;
9410 })
9411
9412 ;;
9413 ;; [vidupq_u_insn])
9414 ;;
9415 (define_insn "mve_vidupq_u<mode>_insn"
9416  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9417        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9418                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9419          VIDUPQ))
9420   (set (match_operand:SI 1 "s_register_operand" "=Te")
9421        (plus:SI (match_dup 2)
9422                 (match_operand:SI 4 "immediate_operand" "i")))]
9423  "TARGET_HAVE_MVE"
9424  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9425
9426 ;;
9427 ;; [vidupq_m_n_u])
9428 ;;
9429 (define_expand "mve_vidupq_m_n_u<mode>"
9430   [(match_operand:MVE_2 0 "s_register_operand")
9431    (match_operand:MVE_2 1 "s_register_operand")
9432    (match_operand:SI 2 "s_register_operand")
9433    (match_operand:SI 3 "mve_imm_selective_upto_8")
9434    (match_operand:HI 4 "vpr_register_operand")]
9435   "TARGET_HAVE_MVE"
9436 {
9437   rtx temp = gen_reg_rtx (SImode);
9438   emit_move_insn (temp, operands[2]);
9439   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9440   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9441                                              operands[2], operands[3],
9442                                              operands[4], inc));
9443   DONE;
9444 })
9445
9446 ;;
9447 ;; [vidupq_m_wb_u_insn])
9448 ;;
9449 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9450  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9451        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9452                       (match_operand:SI 3 "s_register_operand" "2")
9453                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9454                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9455         VIDUPQ_M))
9456   (set (match_operand:SI 2 "s_register_operand" "=Te")
9457        (plus:SI (match_dup 3)
9458                 (match_operand:SI 6 "immediate_operand" "i")))]
9459  "TARGET_HAVE_MVE"
9460  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9461  [(set_attr "length""8")])
9462
9463 ;;
9464 ;; [vddupq_n_u])
9465 ;;
9466 (define_expand "mve_vddupq_n_u<mode>"
9467  [(match_operand:MVE_2 0 "s_register_operand")
9468   (match_operand:SI 1 "s_register_operand")
9469   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9470  "TARGET_HAVE_MVE"
9471 {
9472   rtx temp = gen_reg_rtx (SImode);
9473   emit_move_insn (temp, operands[1]);
9474   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9475   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9476                                           operands[2], inc));
9477   DONE;
9478 })
9479
9480 ;;
9481 ;; [vddupq_u_insn])
9482 ;;
9483 (define_insn "mve_vddupq_u<mode>_insn"
9484  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9485        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9486                       (match_operand:SI 3 "immediate_operand" "i")]
9487         VDDUPQ))
9488   (set (match_operand:SI 1 "s_register_operand" "=Te")
9489        (minus:SI (match_dup 2)
9490                  (match_operand:SI 4 "immediate_operand" "i")))]
9491  "TARGET_HAVE_MVE"
9492  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9493
9494 ;;
9495 ;; [vddupq_m_n_u])
9496 ;;
9497 (define_expand "mve_vddupq_m_n_u<mode>"
9498   [(match_operand:MVE_2 0 "s_register_operand")
9499    (match_operand:MVE_2 1 "s_register_operand")
9500    (match_operand:SI 2 "s_register_operand")
9501    (match_operand:SI 3 "mve_imm_selective_upto_8")
9502    (match_operand:HI 4 "vpr_register_operand")]
9503   "TARGET_HAVE_MVE"
9504 {
9505   rtx temp = gen_reg_rtx (SImode);
9506   emit_move_insn (temp, operands[2]);
9507   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9508   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9509                                              operands[2], operands[3],
9510                                              operands[4], inc));
9511   DONE;
9512 })
9513
9514 ;;
9515 ;; [vddupq_m_wb_u_insn])
9516 ;;
9517 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9518  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9519        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9520                       (match_operand:SI 3 "s_register_operand" "2")
9521                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9522                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9523         VDDUPQ_M))
9524   (set (match_operand:SI 2 "s_register_operand" "=Te")
9525        (minus:SI (match_dup 3)
9526                  (match_operand:SI 6 "immediate_operand" "i")))]
9527  "TARGET_HAVE_MVE"
9528  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9529  [(set_attr "length""8")])
9530
9531 ;;
9532 ;; [vdwdupq_n_u])
9533 ;;
9534 (define_expand "mve_vdwdupq_n_u<mode>"
9535  [(match_operand:MVE_2 0 "s_register_operand")
9536   (match_operand:SI 1 "s_register_operand")
9537   (match_operand:DI 2 "s_register_operand")
9538   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9539  "TARGET_HAVE_MVE"
9540 {
9541   rtx ignore_wb = gen_reg_rtx (SImode);
9542   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9543                                               operands[1], operands[2],
9544                                               operands[3]));
9545   DONE;
9546 })
9547
9548 ;;
9549 ;; [vdwdupq_wb_u])
9550 ;;
9551 (define_expand "mve_vdwdupq_wb_u<mode>"
9552  [(match_operand:SI 0 "s_register_operand")
9553   (match_operand:SI 1 "s_register_operand")
9554   (match_operand:DI 2 "s_register_operand")
9555   (match_operand:SI 3 "mve_imm_selective_upto_8")
9556   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9557  "TARGET_HAVE_MVE"
9558 {
9559   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9560   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9561                                               operands[1], operands[2],
9562                                               operands[3]));
9563   DONE;
9564 })
9565
9566 ;;
9567 ;; [vdwdupq_wb_u_insn])
9568 ;;
9569 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9570   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9571         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9572                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9573                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9574          VDWDUPQ))
9575    (set (match_operand:SI 1 "s_register_operand" "=Te")
9576         (unspec:SI [(match_dup 2)
9577                     (subreg:SI (match_dup 3) 4)
9578                     (match_dup 4)]
9579          VDWDUPQ))]
9580   "TARGET_HAVE_MVE"
9581   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9582 )
9583
9584 ;;
9585 ;; [vdwdupq_m_n_u])
9586 ;;
9587 (define_expand "mve_vdwdupq_m_n_u<mode>"
9588  [(match_operand:MVE_2 0 "s_register_operand")
9589   (match_operand:MVE_2 1 "s_register_operand")
9590   (match_operand:SI 2 "s_register_operand")
9591   (match_operand:DI 3 "s_register_operand")
9592   (match_operand:SI 4 "mve_imm_selective_upto_8")
9593   (match_operand:HI 5 "vpr_register_operand")]
9594  "TARGET_HAVE_MVE"
9595 {
9596   rtx ignore_wb = gen_reg_rtx (SImode);
9597   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9598                                                 operands[1], operands[2],
9599                                                 operands[3], operands[4],
9600                                                 operands[5]));
9601   DONE;
9602 })
9603
9604 ;;
9605 ;; [vdwdupq_m_wb_u])
9606 ;;
9607 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9608  [(match_operand:SI 0 "s_register_operand")
9609   (match_operand:MVE_2 1 "s_register_operand")
9610   (match_operand:SI 2 "s_register_operand")
9611   (match_operand:DI 3 "s_register_operand")
9612   (match_operand:SI 4 "mve_imm_selective_upto_8")
9613   (match_operand:HI 5 "vpr_register_operand")]
9614  "TARGET_HAVE_MVE"
9615 {
9616   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9617   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9618                                                 operands[1], operands[2],
9619                                                 operands[3], operands[4],
9620                                                 operands[5]));
9621   DONE;
9622 })
9623
9624 ;;
9625 ;; [vdwdupq_m_wb_u_insn])
9626 ;;
9627 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9628   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9629         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9630                        (match_operand:SI 3 "s_register_operand" "1")
9631                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9632                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9633                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9634          VDWDUPQ_M))
9635    (set (match_operand:SI 1 "s_register_operand" "=Te")
9636         (unspec:SI [(match_dup 2)
9637                     (match_dup 3)
9638                     (subreg:SI (match_dup 4) 4)
9639                     (match_dup 5)
9640                     (match_dup 6)]
9641          VDWDUPQ_M))
9642   ]
9643   "TARGET_HAVE_MVE"
9644   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9645   [(set_attr "type" "mve_move")
9646    (set_attr "length""8")])
9647
9648 ;;
9649 ;; [viwdupq_n_u])
9650 ;;
9651 (define_expand "mve_viwdupq_n_u<mode>"
9652  [(match_operand:MVE_2 0 "s_register_operand")
9653   (match_operand:SI 1 "s_register_operand")
9654   (match_operand:DI 2 "s_register_operand")
9655   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9656  "TARGET_HAVE_MVE"
9657 {
9658   rtx ignore_wb = gen_reg_rtx (SImode);
9659   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9660                                               operands[1], operands[2],
9661                                               operands[3]));
9662   DONE;
9663 })
9664
9665 ;;
9666 ;; [viwdupq_wb_u])
9667 ;;
9668 (define_expand "mve_viwdupq_wb_u<mode>"
9669  [(match_operand:SI 0 "s_register_operand")
9670   (match_operand:SI 1 "s_register_operand")
9671   (match_operand:DI 2 "s_register_operand")
9672   (match_operand:SI 3 "mve_imm_selective_upto_8")
9673   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9674  "TARGET_HAVE_MVE"
9675 {
9676   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9677   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9678                                               operands[1], operands[2],
9679                                               operands[3]));
9680   DONE;
9681 })
9682
9683 ;;
9684 ;; [viwdupq_wb_u_insn])
9685 ;;
9686 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9687   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9688         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9689                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9690                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9691          VIWDUPQ))
9692    (set (match_operand:SI 1 "s_register_operand" "=Te")
9693         (unspec:SI [(match_dup 2)
9694                     (subreg:SI (match_dup 3) 4)
9695                     (match_dup 4)]
9696          VIWDUPQ))]
9697   "TARGET_HAVE_MVE"
9698   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9699 )
9700
9701 ;;
9702 ;; [viwdupq_m_n_u])
9703 ;;
9704 (define_expand "mve_viwdupq_m_n_u<mode>"
9705  [(match_operand:MVE_2 0 "s_register_operand")
9706   (match_operand:MVE_2 1 "s_register_operand")
9707   (match_operand:SI 2 "s_register_operand")
9708   (match_operand:DI 3 "s_register_operand")
9709   (match_operand:SI 4 "mve_imm_selective_upto_8")
9710   (match_operand:HI 5 "vpr_register_operand")]
9711  "TARGET_HAVE_MVE"
9712 {
9713   rtx ignore_wb = gen_reg_rtx (SImode);
9714   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9715                                                 operands[1], operands[2],
9716                                                 operands[3], operands[4],
9717                                                 operands[5]));
9718   DONE;
9719 })
9720
9721 ;;
9722 ;; [viwdupq_m_wb_u])
9723 ;;
9724 (define_expand "mve_viwdupq_m_wb_u<mode>"
9725  [(match_operand:SI 0 "s_register_operand")
9726   (match_operand:MVE_2 1 "s_register_operand")
9727   (match_operand:SI 2 "s_register_operand")
9728   (match_operand:DI 3 "s_register_operand")
9729   (match_operand:SI 4 "mve_imm_selective_upto_8")
9730   (match_operand:HI 5 "vpr_register_operand")]
9731  "TARGET_HAVE_MVE"
9732 {
9733   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9734   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9735                                                 operands[1], operands[2],
9736                                                 operands[3], operands[4],
9737                                                 operands[5]));
9738   DONE;
9739 })
9740
9741 ;;
9742 ;; [viwdupq_m_wb_u_insn])
9743 ;;
9744 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9745   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9746         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9747                        (match_operand:SI 3 "s_register_operand" "1")
9748                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9749                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9750                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9751          VIWDUPQ_M))
9752    (set (match_operand:SI 1 "s_register_operand" "=Te")
9753         (unspec:SI [(match_dup 2)
9754                     (match_dup 3)
9755                     (subreg:SI (match_dup 4) 4)
9756                     (match_dup 5)
9757                     (match_dup 6)]
9758          VIWDUPQ_M))
9759   ]
9760   "TARGET_HAVE_MVE"
9761   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9762   [(set_attr "type" "mve_move")
9763    (set_attr "length""8")])
9764
9765 ;;
9766 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9767 ;;
9768 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9769   [(set (mem:BLK (scratch))
9770         (unspec:BLK
9771                 [(match_operand:V4SI 1 "s_register_operand" "0")
9772                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9773                  (match_operand:V4SI 3 "s_register_operand" "w")]
9774          VSTRWSBWBQ))
9775    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9776         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9777          VSTRWSBWBQ))
9778   ]
9779   "TARGET_HAVE_MVE"
9780 {
9781    rtx ops[3];
9782    ops[0] = operands[1];
9783    ops[1] = operands[2];
9784    ops[2] = operands[3];
9785    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9786    return "";
9787 }
9788   [(set_attr "length" "4")])
9789
9790 ;;
9791 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9792 ;;
9793 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9794  [(set (mem:BLK (scratch))
9795        (unspec:BLK
9796                 [(match_operand:V4SI 1 "s_register_operand" "0")
9797                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9798                  (match_operand:V4SI 3 "s_register_operand" "w")
9799                  (match_operand:HI 4 "vpr_register_operand")]
9800         VSTRWSBWBQ))
9801    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9802         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9803          VSTRWSBWBQ))
9804   ]
9805   "TARGET_HAVE_MVE"
9806 {
9807    rtx ops[3];
9808    ops[0] = operands[1];
9809    ops[1] = operands[2];
9810    ops[2] = operands[3];
9811    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9812    return "";
9813 }
9814   [(set_attr "length" "8")])
9815
9816 ;;
9817 ;; [vstrwq_scatter_base_wb_f]
9818 ;;
9819 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9820  [(set (mem:BLK (scratch))
9821        (unspec:BLK
9822                 [(match_operand:V4SI 1 "s_register_operand" "0")
9823                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9824                  (match_operand:V4SF 3 "s_register_operand" "w")]
9825          VSTRWQSBWB_F))
9826    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9827         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9828          VSTRWQSBWB_F))
9829   ]
9830   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9831 {
9832    rtx ops[3];
9833    ops[0] = operands[1];
9834    ops[1] = operands[2];
9835    ops[2] = operands[3];
9836    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9837    return "";
9838 }
9839   [(set_attr "length" "4")])
9840
9841 ;;
9842 ;; [vstrwq_scatter_base_wb_p_f]
9843 ;;
9844 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9845  [(set (mem:BLK (scratch))
9846        (unspec:BLK
9847                 [(match_operand:V4SI 1 "s_register_operand" "0")
9848                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9849                  (match_operand:V4SF 3 "s_register_operand" "w")
9850                  (match_operand:HI 4 "vpr_register_operand")]
9851         VSTRWQSBWB_F))
9852    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9853         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9854          VSTRWQSBWB_F))
9855   ]
9856   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9857 {
9858    rtx ops[3];
9859    ops[0] = operands[1];
9860    ops[1] = operands[2];
9861    ops[2] = operands[3];
9862    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9863    return "";
9864 }
9865   [(set_attr "length" "8")])
9866
9867 ;;
9868 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9869 ;;
9870 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9871   [(set (mem:BLK (scratch))
9872         (unspec:BLK
9873                 [(match_operand:V2DI 1 "s_register_operand" "0")
9874                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9875                  (match_operand:V2DI 3 "s_register_operand" "w")]
9876          VSTRDSBWBQ))
9877    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9878         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9879          VSTRDSBWBQ))
9880   ]
9881   "TARGET_HAVE_MVE"
9882 {
9883    rtx ops[3];
9884    ops[0] = operands[1];
9885    ops[1] = operands[2];
9886    ops[2] = operands[3];
9887    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9888    return "";
9889 }
9890   [(set_attr "length" "4")])
9891
9892 ;;
9893 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9894 ;;
9895 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9896   [(set (mem:BLK (scratch))
9897         (unspec:BLK
9898                 [(match_operand:V2DI 1 "s_register_operand" "0")
9899                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9900                  (match_operand:V2DI 3 "s_register_operand" "w")
9901                  (match_operand:HI 4 "vpr_register_operand")]
9902          VSTRDSBWBQ))
9903    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9904         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9905          VSTRDSBWBQ))
9906   ]
9907   "TARGET_HAVE_MVE"
9908 {
9909    rtx ops[3];
9910    ops[0] = operands[1];
9911    ops[1] = operands[2];
9912    ops[2] = operands[3];
9913    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9914    return "";
9915 }
9916   [(set_attr "length" "8")])
9917
9918 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9919   [(match_operand:V4SI 0 "s_register_operand")
9920    (match_operand:V4SI 1 "s_register_operand")
9921    (match_operand:SI 2 "mve_vldrd_immediate")
9922    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9923   "TARGET_HAVE_MVE"
9924 {
9925   rtx ignore_result = gen_reg_rtx (V4SImode);
9926   emit_insn (
9927   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9928                                                  operands[1], operands[2]));
9929   DONE;
9930 })
9931
9932 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9933   [(match_operand:V4SI 0 "s_register_operand")
9934    (match_operand:V4SI 1 "s_register_operand")
9935    (match_operand:SI 2 "mve_vldrd_immediate")
9936    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9937   "TARGET_HAVE_MVE"
9938 {
9939   rtx ignore_wb = gen_reg_rtx (V4SImode);
9940   emit_insn (
9941   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9942                                                  operands[1], operands[2]));
9943   DONE;
9944 })
9945
9946 ;;
9947 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9948 ;;
9949 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9950   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9951         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9952                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9953                       (mem:BLK (scratch))]
9954          VLDRWGBWBQ))
9955    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9956         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9957          VLDRWGBWBQ))
9958   ]
9959   "TARGET_HAVE_MVE"
9960 {
9961    rtx ops[3];
9962    ops[0] = operands[0];
9963    ops[1] = operands[2];
9964    ops[2] = operands[3];
9965    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9966    return "";
9967 }
9968   [(set_attr "length" "4")])
9969
9970 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9971   [(match_operand:V4SI 0 "s_register_operand")
9972    (match_operand:V4SI 1 "s_register_operand")
9973    (match_operand:SI 2 "mve_vldrd_immediate")
9974    (match_operand:HI 3 "vpr_register_operand")
9975    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9976   "TARGET_HAVE_MVE"
9977 {
9978   rtx ignore_result = gen_reg_rtx (V4SImode);
9979   emit_insn (
9980   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9981                                                    operands[1], operands[2],
9982                                                    operands[3]));
9983   DONE;
9984 })
9985 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9986   [(match_operand:V4SI 0 "s_register_operand")
9987    (match_operand:V4SI 1 "s_register_operand")
9988    (match_operand:SI 2 "mve_vldrd_immediate")
9989    (match_operand:HI 3 "vpr_register_operand")
9990    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9991   "TARGET_HAVE_MVE"
9992 {
9993   rtx ignore_wb = gen_reg_rtx (V4SImode);
9994   emit_insn (
9995   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9996                                                    operands[1], operands[2],
9997                                                    operands[3]));
9998   DONE;
9999 })
10000
10001 ;;
10002 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10003 ;;
10004 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10005   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10006         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10007                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10008                       (match_operand:HI 4 "vpr_register_operand" "Up")
10009                       (mem:BLK (scratch))]
10010          VLDRWGBWBQ))
10011    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10012         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10013          VLDRWGBWBQ))
10014   ]
10015   "TARGET_HAVE_MVE"
10016 {
10017    rtx ops[3];
10018    ops[0] = operands[0];
10019    ops[1] = operands[2];
10020    ops[2] = operands[3];
10021    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10022    return "";
10023 }
10024   [(set_attr "length" "8")])
10025
10026 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10027   [(match_operand:V4SI 0 "s_register_operand")
10028    (match_operand:V4SI 1 "s_register_operand")
10029    (match_operand:SI 2 "mve_vldrd_immediate")
10030    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10031   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10032 {
10033   rtx ignore_result = gen_reg_rtx (V4SFmode);
10034   emit_insn (
10035   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10036                                             operands[1], operands[2]));
10037   DONE;
10038 })
10039
10040 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10041   [(match_operand:V4SF 0 "s_register_operand")
10042    (match_operand:V4SI 1 "s_register_operand")
10043    (match_operand:SI 2 "mve_vldrd_immediate")
10044    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10045   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10046 {
10047   rtx ignore_wb = gen_reg_rtx (V4SImode);
10048   emit_insn (
10049   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10050                                             operands[1], operands[2]));
10051   DONE;
10052 })
10053
10054 ;;
10055 ;; [vldrwq_gather_base_wb_f]
10056 ;;
10057 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10058   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10059         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10060                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10061                       (mem:BLK (scratch))]
10062          VLDRWQGBWB_F))
10063    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10064         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10065          VLDRWQGBWB_F))
10066   ]
10067   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10068 {
10069    rtx ops[3];
10070    ops[0] = operands[0];
10071    ops[1] = operands[2];
10072    ops[2] = operands[3];
10073    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10074    return "";
10075 }
10076   [(set_attr "length" "4")])
10077
10078 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10079   [(match_operand:V4SI 0 "s_register_operand")
10080    (match_operand:V4SI 1 "s_register_operand")
10081    (match_operand:SI 2 "mve_vldrd_immediate")
10082    (match_operand:HI 3 "vpr_register_operand")
10083    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10084   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10085 {
10086   rtx ignore_result = gen_reg_rtx (V4SFmode);
10087   emit_insn (
10088   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10089                                               operands[1], operands[2],
10090                                               operands[3]));
10091   DONE;
10092 })
10093
10094 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10095   [(match_operand:V4SF 0 "s_register_operand")
10096    (match_operand:V4SI 1 "s_register_operand")
10097    (match_operand:SI 2 "mve_vldrd_immediate")
10098    (match_operand:HI 3 "vpr_register_operand")
10099    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10100   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10101 {
10102   rtx ignore_wb = gen_reg_rtx (V4SImode);
10103   emit_insn (
10104   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10105                                               operands[1], operands[2],
10106                                               operands[3]));
10107   DONE;
10108 })
10109
10110 ;;
10111 ;; [vldrwq_gather_base_wb_z_f]
10112 ;;
10113 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10114   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10115         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10116                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10117                       (match_operand:HI 4 "vpr_register_operand" "Up")
10118                       (mem:BLK (scratch))]
10119          VLDRWQGBWB_F))
10120    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10121         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10122          VLDRWQGBWB_F))
10123   ]
10124   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10125 {
10126    rtx ops[3];
10127    ops[0] = operands[0];
10128    ops[1] = operands[2];
10129    ops[2] = operands[3];
10130    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10131    return "";
10132 }
10133   [(set_attr "length" "8")])
10134
10135 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10136   [(match_operand:V2DI 0 "s_register_operand")
10137    (match_operand:V2DI 1 "s_register_operand")
10138    (match_operand:SI 2 "mve_vldrd_immediate")
10139    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10140   "TARGET_HAVE_MVE"
10141 {
10142   rtx ignore_result = gen_reg_rtx (V2DImode);
10143   emit_insn (
10144   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10145                                                  operands[1], operands[2]));
10146   DONE;
10147 })
10148
10149 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10150   [(match_operand:V2DI 0 "s_register_operand")
10151    (match_operand:V2DI 1 "s_register_operand")
10152    (match_operand:SI 2 "mve_vldrd_immediate")
10153    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10154   "TARGET_HAVE_MVE"
10155 {
10156   rtx ignore_wb = gen_reg_rtx (V2DImode);
10157   emit_insn (
10158   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10159                                                  operands[1], operands[2]));
10160   DONE;
10161 })
10162
10163
10164 ;;
10165 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10166 ;;
10167 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10168   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10169         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10170                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10171                       (mem:BLK (scratch))]
10172          VLDRDGBWBQ))
10173    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10174         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10175          VLDRDGBWBQ))
10176   ]
10177   "TARGET_HAVE_MVE"
10178 {
10179    rtx ops[3];
10180    ops[0] = operands[0];
10181    ops[1] = operands[2];
10182    ops[2] = operands[3];
10183    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10184    return "";
10185 }
10186   [(set_attr "length" "4")])
10187
10188 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10189   [(match_operand:V2DI 0 "s_register_operand")
10190    (match_operand:V2DI 1 "s_register_operand")
10191    (match_operand:SI 2 "mve_vldrd_immediate")
10192    (match_operand:HI 3 "vpr_register_operand")
10193    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10194   "TARGET_HAVE_MVE"
10195 {
10196   rtx ignore_result = gen_reg_rtx (V2DImode);
10197   emit_insn (
10198   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10199                                                    operands[1], operands[2],
10200                                                    operands[3]));
10201   DONE;
10202 })
10203
10204 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10205   [(match_operand:V2DI 0 "s_register_operand")
10206    (match_operand:V2DI 1 "s_register_operand")
10207    (match_operand:SI 2 "mve_vldrd_immediate")
10208    (match_operand:HI 3 "vpr_register_operand")
10209    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10210   "TARGET_HAVE_MVE"
10211 {
10212   rtx ignore_wb = gen_reg_rtx (V2DImode);
10213   emit_insn (
10214   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10215                                                    operands[1], operands[2],
10216                                                    operands[3]));
10217   DONE;
10218 })
10219
10220 (define_insn "get_fpscr_nzcvqc"
10221  [(set (match_operand:SI 0 "register_operand" "=r")
10222    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10223  "TARGET_HAVE_MVE"
10224  "vmrs\\t%0, FPSCR_nzcvqc"
10225   [(set_attr "type" "mve_move")])
10226
10227 (define_insn "set_fpscr_nzcvqc"
10228  [(set (reg:SI VFPCC_REGNUM)
10229    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10230     VUNSPEC_SET_FPSCR_NZCVQC))]
10231  "TARGET_HAVE_MVE"
10232  "vmsr\\tFPSCR_nzcvqc, %0"
10233   [(set_attr "type" "mve_move")])
10234
10235 ;;
10236 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10237 ;;
10238 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10239   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10240         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10241                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10242                       (match_operand:HI 4 "vpr_register_operand" "Up")
10243                       (mem:BLK (scratch))]
10244          VLDRDGBWBQ))
10245    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10246         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10247          VLDRDGBWBQ))
10248   ]
10249   "TARGET_HAVE_MVE"
10250 {
10251    rtx ops[3];
10252    ops[0] = operands[0];
10253    ops[1] = operands[2];
10254    ops[2] = operands[3];
10255    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10256    return "";
10257 }
10258   [(set_attr "length" "8")])
10259 ;;
10260 ;; [vadciq_m_s, vadciq_m_u])
10261 ;;
10262 (define_insn "mve_vadciq_m_<supf>v4si"
10263   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10264         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10265                       (match_operand:V4SI 2 "s_register_operand" "w")
10266                       (match_operand:V4SI 3 "s_register_operand" "w")
10267                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10268          VADCIQ_M))
10269    (set (reg:SI VFPCC_REGNUM)
10270         (unspec:SI [(const_int 0)]
10271          VADCIQ_M))
10272   ]
10273   "TARGET_HAVE_MVE"
10274   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10275   [(set_attr "type" "mve_move")
10276    (set_attr "length" "8")])
10277
10278 ;;
10279 ;; [vadciq_u, vadciq_s])
10280 ;;
10281 (define_insn "mve_vadciq_<supf>v4si"
10282   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10283         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10284                       (match_operand:V4SI 2 "s_register_operand" "w")]
10285          VADCIQ))
10286    (set (reg:SI VFPCC_REGNUM)
10287         (unspec:SI [(const_int 0)]
10288          VADCIQ))
10289   ]
10290   "TARGET_HAVE_MVE"
10291   "vadci.i32\t%q0, %q1, %q2"
10292   [(set_attr "type" "mve_move")
10293    (set_attr "length" "4")])
10294
10295 ;;
10296 ;; [vadcq_m_s, vadcq_m_u])
10297 ;;
10298 (define_insn "mve_vadcq_m_<supf>v4si"
10299   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10300         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10301                       (match_operand:V4SI 2 "s_register_operand" "w")
10302                       (match_operand:V4SI 3 "s_register_operand" "w")
10303                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10304          VADCQ_M))
10305    (set (reg:SI VFPCC_REGNUM)
10306         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10307          VADCQ_M))
10308   ]
10309   "TARGET_HAVE_MVE"
10310   "vpst\;vadct.i32\t%q0, %q2, %q3"
10311   [(set_attr "type" "mve_move")
10312    (set_attr "length" "8")])
10313
10314 ;;
10315 ;; [vadcq_u, vadcq_s])
10316 ;;
10317 (define_insn "mve_vadcq_<supf>v4si"
10318   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10319         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10320                        (match_operand:V4SI 2 "s_register_operand" "w")]
10321          VADCQ))
10322    (set (reg:SI VFPCC_REGNUM)
10323         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10324          VADCQ))
10325   ]
10326   "TARGET_HAVE_MVE"
10327   "vadc.i32\t%q0, %q1, %q2"
10328   [(set_attr "type" "mve_move")
10329    (set_attr "length" "4")
10330    (set_attr "conds" "set")])
10331
10332 ;;
10333 ;; [vsbciq_m_u, vsbciq_m_s])
10334 ;;
10335 (define_insn "mve_vsbciq_m_<supf>v4si"
10336   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10337         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10338                       (match_operand:V4SI 2 "s_register_operand" "w")
10339                       (match_operand:V4SI 3 "s_register_operand" "w")
10340                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10341          VSBCIQ_M))
10342    (set (reg:SI VFPCC_REGNUM)
10343         (unspec:SI [(const_int 0)]
10344          VSBCIQ_M))
10345   ]
10346   "TARGET_HAVE_MVE"
10347   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10348   [(set_attr "type" "mve_move")
10349    (set_attr "length" "8")])
10350
10351 ;;
10352 ;; [vsbciq_s, vsbciq_u])
10353 ;;
10354 (define_insn "mve_vsbciq_<supf>v4si"
10355   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10356         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10357                       (match_operand:V4SI 2 "s_register_operand" "w")]
10358          VSBCIQ))
10359    (set (reg:SI VFPCC_REGNUM)
10360         (unspec:SI [(const_int 0)]
10361          VSBCIQ))
10362   ]
10363   "TARGET_HAVE_MVE"
10364   "vsbci.i32\t%q0, %q1, %q2"
10365   [(set_attr "type" "mve_move")
10366    (set_attr "length" "4")])
10367
10368 ;;
10369 ;; [vsbcq_m_u, vsbcq_m_s])
10370 ;;
10371 (define_insn "mve_vsbcq_m_<supf>v4si"
10372   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10373         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10374                       (match_operand:V4SI 2 "s_register_operand" "w")
10375                       (match_operand:V4SI 3 "s_register_operand" "w")
10376                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10377          VSBCQ_M))
10378    (set (reg:SI VFPCC_REGNUM)
10379         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10380          VSBCQ_M))
10381   ]
10382   "TARGET_HAVE_MVE"
10383   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10384   [(set_attr "type" "mve_move")
10385    (set_attr "length" "8")])
10386
10387 ;;
10388 ;; [vsbcq_s, vsbcq_u])
10389 ;;
10390 (define_insn "mve_vsbcq_<supf>v4si"
10391   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10392         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10393                       (match_operand:V4SI 2 "s_register_operand" "w")]
10394          VSBCQ))
10395    (set (reg:SI VFPCC_REGNUM)
10396         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10397          VSBCQ))
10398   ]
10399   "TARGET_HAVE_MVE"
10400   "vsbc.i32\t%q0, %q1, %q2"
10401   [(set_attr "type" "mve_move")
10402    (set_attr "length" "4")])
10403
10404 ;;
10405 ;; [vst2q])
10406 ;;
10407 (define_insn "mve_vst2q<mode>"
10408   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10409         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10410                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10411          VST2Q))
10412   ]
10413   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10414    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10415 {
10416    rtx ops[4];
10417    int regno = REGNO (operands[1]);
10418    ops[0] = gen_rtx_REG (TImode, regno);
10419    ops[1] = gen_rtx_REG (TImode, regno + 4);
10420    rtx reg  = operands[0];
10421    while (reg && !REG_P (reg))
10422     reg = XEXP (reg, 0);
10423    gcc_assert (REG_P (reg));
10424    ops[2] = reg;
10425    ops[3] = operands[0];
10426    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10427                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10428    return "";
10429 }
10430   [(set_attr "length" "8")])
10431
10432 ;;
10433 ;; [vld2q])
10434 ;;
10435 (define_insn "mve_vld2q<mode>"
10436   [(set (match_operand:OI 0 "s_register_operand" "=w")
10437         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10438                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10439          VLD2Q))
10440   ]
10441   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10442    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10443 {
10444    rtx ops[4];
10445    int regno = REGNO (operands[0]);
10446    ops[0] = gen_rtx_REG (TImode, regno);
10447    ops[1] = gen_rtx_REG (TImode, regno + 4);
10448    rtx reg  = operands[1];
10449    while (reg && !REG_P (reg))
10450     reg = XEXP (reg, 0);
10451    gcc_assert (REG_P (reg));
10452    ops[2] = reg;
10453    ops[3] = operands[1];
10454    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10455                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10456    return "";
10457 }
10458   [(set_attr "length" "8")])
10459
10460 ;;
10461 ;; [vld4q])
10462 ;;
10463 (define_insn "mve_vld4q<mode>"
10464   [(set (match_operand:XI 0 "s_register_operand" "=w")
10465         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10466                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10467          VLD4Q))
10468   ]
10469   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10470    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10471 {
10472    rtx ops[6];
10473    int regno = REGNO (operands[0]);
10474    ops[0] = gen_rtx_REG (TImode, regno);
10475    ops[1] = gen_rtx_REG (TImode, regno+4);
10476    ops[2] = gen_rtx_REG (TImode, regno+8);
10477    ops[3] = gen_rtx_REG (TImode, regno + 12);
10478    rtx reg  = operands[1];
10479    while (reg && !REG_P (reg))
10480     reg = XEXP (reg, 0);
10481    gcc_assert (REG_P (reg));
10482    ops[4] = reg;
10483    ops[5] = operands[1];
10484    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10485                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10486                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10487                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10488    return "";
10489 }
10490   [(set_attr "length" "16")])
10491 ;;
10492 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10493 ;;
10494 (define_insn "mve_vec_extract<mode><V_elem_l>"
10495  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10496    (vec_select:<V_elem>
10497     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10498     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10499   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10500    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10501 {
10502   if (BYTES_BIG_ENDIAN)
10503     {
10504       int elt = INTVAL (operands[2]);
10505       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10506       operands[2] = GEN_INT (elt);
10507     }
10508   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10509 }
10510  [(set_attr "type" "mve_move")])
10511
10512 (define_insn "mve_vec_extractv2didi"
10513  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10514    (vec_select:DI
10515     (match_operand:V2DI 1 "s_register_operand" "w")
10516     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10517   "TARGET_HAVE_MVE"
10518 {
10519   int elt = INTVAL (operands[2]);
10520   if (BYTES_BIG_ENDIAN)
10521     elt = 1 - elt;
10522
10523   if (elt == 0)
10524    return "vmov\t%Q0, %R0, %e1";
10525   else
10526    return "vmov\t%Q0, %R0, %f1";
10527 }
10528  [(set_attr "type" "mve_move")])
10529
10530 (define_insn "*mve_vec_extract_sext_internal<mode>"
10531  [(set (match_operand:SI 0 "s_register_operand" "=r")
10532    (sign_extend:SI
10533     (vec_select:<V_elem>
10534      (match_operand:MVE_2 1 "s_register_operand" "w")
10535      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10536   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10537    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10538 {
10539   if (BYTES_BIG_ENDIAN)
10540     {
10541       int elt = INTVAL (operands[2]);
10542       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10543       operands[2] = GEN_INT (elt);
10544     }
10545   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10546 }
10547  [(set_attr "type" "mve_move")])
10548
10549 (define_insn "*mve_vec_extract_zext_internal<mode>"
10550  [(set (match_operand:SI 0 "s_register_operand" "=r")
10551    (zero_extend:SI
10552     (vec_select:<V_elem>
10553      (match_operand:MVE_2 1 "s_register_operand" "w")
10554      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10555   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10556    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10557 {
10558   if (BYTES_BIG_ENDIAN)
10559     {
10560       int elt = INTVAL (operands[2]);
10561       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10562       operands[2] = GEN_INT (elt);
10563     }
10564   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10565 }
10566  [(set_attr "type" "mve_move")])
10567
10568 ;;
10569 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10570 ;;
10571 (define_insn "mve_vec_set<mode>_internal"
10572  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10573        (vec_merge:VQ2
10574         (vec_duplicate:VQ2
10575           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10576         (match_operand:VQ2 3 "s_register_operand" "0")
10577         (match_operand:SI 2 "immediate_operand" "i")))]
10578   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10579    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10580 {
10581   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10582   if (BYTES_BIG_ENDIAN)
10583     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10584   operands[2] = GEN_INT (elt);
10585
10586   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10587 }
10588  [(set_attr "type" "mve_move")])
10589
10590 (define_insn "mve_vec_setv2di_internal"
10591  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10592        (vec_merge:V2DI
10593         (vec_duplicate:V2DI
10594           (match_operand:DI 1 "nonimmediate_operand" "r"))
10595         (match_operand:V2DI 3 "s_register_operand" "0")
10596         (match_operand:SI 2 "immediate_operand" "i")))]
10597  "TARGET_HAVE_MVE"
10598 {
10599   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10600   if (BYTES_BIG_ENDIAN)
10601     elt = 1 - elt;
10602
10603   if (elt == 0)
10604    return "vmov\t%e0, %Q1, %R1";
10605   else
10606    return "vmov\t%f0, %J1, %K1";
10607 }
10608  [(set_attr "type" "mve_move")])
10609
10610 ;;
10611 ;; [uqrshll_di]
10612 ;;
10613 (define_insn "mve_uqrshll_sat<supf>_di"
10614   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10615         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10616                     (match_operand:SI 2 "register_operand" "r")]
10617          UQRSHLLQ))]
10618   "TARGET_HAVE_MVE"
10619   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10620   [(set_attr "predicable" "yes")])
10621
10622 ;;
10623 ;; [sqrshrl_di]
10624 ;;
10625 (define_insn "mve_sqrshrl_sat<supf>_di"
10626   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10627         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10628                     (match_operand:SI 2 "register_operand" "r")]
10629          SQRSHRLQ))]
10630   "TARGET_HAVE_MVE"
10631   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10632   [(set_attr "predicable" "yes")])
10633
10634 ;;
10635 ;; [uqrshl_si]
10636 ;;
10637 (define_insn "mve_uqrshl_si"
10638   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10639         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10640                     (match_operand:SI 2 "register_operand" "r")]
10641          UQRSHL))]
10642   "TARGET_HAVE_MVE"
10643   "uqrshl%?\\t%1, %2"
10644   [(set_attr "predicable" "yes")])
10645
10646 ;;
10647 ;; [sqrshr_si]
10648 ;;
10649 (define_insn "mve_sqrshr_si"
10650   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10651         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10652                     (match_operand:SI 2 "register_operand" "r")]
10653          SQRSHR))]
10654   "TARGET_HAVE_MVE"
10655   "sqrshr%?\\t%1, %2"
10656   [(set_attr "predicable" "yes")])
10657
10658 ;;
10659 ;; [uqshll_di]
10660 ;;
10661 (define_insn "mve_uqshll_di"
10662   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10663         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10664                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10665   "TARGET_HAVE_MVE"
10666   "uqshll%?\\t%Q1, %R1, %2"
10667   [(set_attr "predicable" "yes")])
10668
10669 ;;
10670 ;; [urshrl_di]
10671 ;;
10672 (define_insn "mve_urshrl_di"
10673   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10674         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10675                     (match_operand:SI 2 "immediate_operand" "Pg")]
10676          URSHRL))]
10677   "TARGET_HAVE_MVE"
10678   "urshrl%?\\t%Q1, %R1, %2"
10679   [(set_attr "predicable" "yes")])
10680
10681 ;;
10682 ;; [uqshl_si]
10683 ;;
10684 (define_insn "mve_uqshl_si"
10685   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10686         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10687                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10688   "TARGET_HAVE_MVE"
10689   "uqshl%?\\t%1, %2"
10690   [(set_attr "predicable" "yes")])
10691
10692 ;;
10693 ;; [urshr_si]
10694 ;;
10695 (define_insn "mve_urshr_si"
10696   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10697         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10698                     (match_operand:SI 2 "immediate_operand" "Pg")]
10699          URSHR))]
10700   "TARGET_HAVE_MVE"
10701   "urshr%?\\t%1, %2"
10702   [(set_attr "predicable" "yes")])
10703
10704 ;;
10705 ;; [sqshl_si]
10706 ;;
10707 (define_insn "mve_sqshl_si"
10708   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10709         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10710                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10711   "TARGET_HAVE_MVE"
10712   "sqshl%?\\t%1, %2"
10713   [(set_attr "predicable" "yes")])
10714
10715 ;;
10716 ;; [srshr_si]
10717 ;;
10718 (define_insn "mve_srshr_si"
10719   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10720         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10721                     (match_operand:SI 2 "immediate_operand" "Pg")]
10722          SRSHR))]
10723   "TARGET_HAVE_MVE"
10724   "srshr%?\\t%1, %2"
10725   [(set_attr "predicable" "yes")])
10726
10727 ;;
10728 ;; [srshrl_di]
10729 ;;
10730 (define_insn "mve_srshrl_di"
10731   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10732         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10733                     (match_operand:SI 2 "immediate_operand" "Pg")]
10734          SRSHRL))]
10735   "TARGET_HAVE_MVE"
10736   "srshrl%?\\t%Q1, %R1, %2"
10737   [(set_attr "predicable" "yes")])
10738
10739 ;;
10740 ;; [sqshll_di]
10741 ;;
10742 (define_insn "mve_sqshll_di"
10743   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10744         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10745                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10746   "TARGET_HAVE_MVE"
10747   "sqshll%?\\t%Q1, %R1, %2"
10748   [(set_attr "predicable" "yes")])
10749
10750 ;;
10751 ;; [vshlcq_m_u vshlcq_m_s]
10752 ;;
10753 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10754  [(match_operand:MVE_2 0 "s_register_operand")
10755   (match_operand:MVE_2 1 "s_register_operand")
10756   (match_operand:SI 2 "s_register_operand")
10757   (match_operand:SI 3 "mve_imm_32")
10758   (match_operand:HI 4 "vpr_register_operand")
10759   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10760  "TARGET_HAVE_MVE"
10761 {
10762   rtx ignore_wb = gen_reg_rtx (SImode);
10763   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10764                                             operands[2], operands[3],
10765                                             operands[4]));
10766   DONE;
10767 })
10768
10769 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10770  [(match_operand:SI 0 "s_register_operand")
10771   (match_operand:MVE_2 1 "s_register_operand")
10772   (match_operand:SI 2 "s_register_operand")
10773   (match_operand:SI 3 "mve_imm_32")
10774   (match_operand:HI 4 "vpr_register_operand")
10775   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10776  "TARGET_HAVE_MVE"
10777 {
10778   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10779   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10780                                             operands[1], operands[2],
10781                                             operands[3], operands[4]));
10782   DONE;
10783 })
10784
10785 (define_insn "mve_vshlcq_m_<supf><mode>"
10786  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10787        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10788                       (match_operand:SI 3 "s_register_operand" "1")
10789                       (match_operand:SI 4 "mve_imm_32" "Rf")
10790                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10791         VSHLCQ_M))
10792   (set (match_operand:SI  1 "s_register_operand" "=r")
10793        (unspec:SI [(match_dup 2)
10794                    (match_dup 3)
10795                    (match_dup 4)
10796                    (match_dup 5)]
10797         VSHLCQ_M))
10798  ]
10799  "TARGET_HAVE_MVE"
10800  "vpst\;vshlct\t%q0, %1, %4"
10801  [(set_attr "type" "mve_move")
10802   (set_attr "length" "8")])
10803
10804 (define_insn "*mve_vec_duplicate<mode>"
10805  [(set (match_operand:MVE_VLD_ST 0 "s_register_operand" "=w")
10806        (vec_duplicate:MVE_VLD_ST (match_operand:<V_elem> 1 "general_operand" "r")))]
10807  "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
10808  "vdup.<V_sz_elem>\t%q0, %1"
10809  [(set_attr "type" "mve_move")])
10810
10811 ;; CDE instructions on MVE registers.
10812
10813 (define_insn "arm_vcx1qv16qi"
10814   [(set (match_operand:V16QI 0 "register_operand" "=t")
10815         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10816                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10817          UNSPEC_VCDE))]
10818   "TARGET_CDE && TARGET_HAVE_MVE"
10819   "vcx1\\tp%c1, %q0, #%c2"
10820   [(set_attr "type" "coproc")]
10821 )
10822
10823 (define_insn "arm_vcx1qav16qi"
10824   [(set (match_operand:V16QI 0 "register_operand" "=t")
10825         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10826                             (match_operand:V16QI 2 "register_operand" "0")
10827                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10828          UNSPEC_VCDEA))]
10829   "TARGET_CDE && TARGET_HAVE_MVE"
10830   "vcx1a\\tp%c1, %q0, #%c3"
10831   [(set_attr "type" "coproc")]
10832 )
10833
10834 (define_insn "arm_vcx2qv16qi"
10835   [(set (match_operand:V16QI 0 "register_operand" "=t")
10836         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10837                           (match_operand:V16QI 2 "register_operand" "t")
10838                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10839          UNSPEC_VCDE))]
10840   "TARGET_CDE && TARGET_HAVE_MVE"
10841   "vcx2\\tp%c1, %q0, %q2, #%c3"
10842   [(set_attr "type" "coproc")]
10843 )
10844
10845 (define_insn "arm_vcx2qav16qi"
10846   [(set (match_operand:V16QI 0 "register_operand" "=t")
10847         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10848                           (match_operand:V16QI 2 "register_operand" "0")
10849                           (match_operand:V16QI 3 "register_operand" "t")
10850                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10851          UNSPEC_VCDEA))]
10852   "TARGET_CDE && TARGET_HAVE_MVE"
10853   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10854   [(set_attr "type" "coproc")]
10855 )
10856
10857 (define_insn "arm_vcx3qv16qi"
10858   [(set (match_operand:V16QI 0 "register_operand" "=t")
10859         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10860                           (match_operand:V16QI 2 "register_operand" "t")
10861                           (match_operand:V16QI 3 "register_operand" "t")
10862                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10863          UNSPEC_VCDE))]
10864   "TARGET_CDE && TARGET_HAVE_MVE"
10865   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10866   [(set_attr "type" "coproc")]
10867 )
10868
10869 (define_insn "arm_vcx3qav16qi"
10870   [(set (match_operand:V16QI 0 "register_operand" "=t")
10871         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10872                           (match_operand:V16QI 2 "register_operand" "0")
10873                           (match_operand:V16QI 3 "register_operand" "t")
10874                           (match_operand:V16QI 4 "register_operand" "t")
10875                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10876          UNSPEC_VCDEA))]
10877   "TARGET_CDE && TARGET_HAVE_MVE"
10878   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10879   [(set_attr "type" "coproc")]
10880 )
10881
10882 (define_insn "arm_vcx1q<a>_p_v16qi"
10883   [(set (match_operand:V16QI 0 "register_operand" "=t")
10884         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10885                            (match_operand:V16QI 2 "register_operand" "0")
10886                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10887                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10888          CDE_VCX))]
10889   "TARGET_CDE && TARGET_HAVE_MVE"
10890   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10891   [(set_attr "type" "coproc")
10892    (set_attr "length" "8")]
10893 )
10894
10895 (define_insn "arm_vcx2q<a>_p_v16qi"
10896   [(set (match_operand:V16QI 0 "register_operand" "=t")
10897         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10898                           (match_operand:V16QI 2 "register_operand" "0")
10899                           (match_operand:V16QI 3 "register_operand" "t")
10900                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10901                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10902          CDE_VCX))]
10903   "TARGET_CDE && TARGET_HAVE_MVE"
10904   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10905   [(set_attr "type" "coproc")
10906    (set_attr "length" "8")]
10907 )
10908
10909 (define_insn "arm_vcx3q<a>_p_v16qi"
10910   [(set (match_operand:V16QI 0 "register_operand" "=t")
10911         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10912                           (match_operand:V16QI 2 "register_operand" "0")
10913                           (match_operand:V16QI 3 "register_operand" "t")
10914                           (match_operand:V16QI 4 "register_operand" "t")
10915                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10916                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10917          CDE_VCX))]
10918   "TARGET_CDE && TARGET_HAVE_MVE"
10919   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10920   [(set_attr "type" "coproc")
10921    (set_attr "length" "8")]
10922 )