gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2021 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_insn "*mve_mov<mode>"
  21   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Ux,w")
  22         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,UxUi,r,Dm,w,Ul"))]
  23   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  24 {
  25   if (which_alternative == 3 || which_alternative == 6)
  26     {
  27       int width, is_valid;
  28       static char templ[40];
  29
  30       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
  31         &operands[1], &width);
  32
  33       gcc_assert (is_valid != 0);
  34
  35       if (width == 0)
  36         return "vmov.f32\t%q0, %1  @ <mode>";
  37       else
  38         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
  39       return templ;
  40     }
  41
  42   if (which_alternative == 4 || which_alternative == 7)
  43     {
  44       if (<MODE>mode == V2DFmode || <MODE>mode == V2DImode || <MODE>mode == TImode)
  45         {
  46           if (which_alternative == 7)
  47             output_asm_insn ("vstrw.32\t%q1, %E0", operands);
  48           else
  49             output_asm_insn ("vldrw.u32\t%q0, %E1",operands);
  50         }
  51       else
  52         {
  53           if (which_alternative == 7)
  54             output_asm_insn ("vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0", operands);
  55           else
  56             output_asm_insn ("vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1", operands);
  57         }
  58       return "";
  59     }
  60   switch (which_alternative)
  61     {
  62     case 0:
  63       return "vmov\t%q0, %q1";
  64     case 1:
  65       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
  66     case 2:
  67       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
  68     case 5:
  69       return output_move_quad (operands);
  70     case 8:
  71         return output_move_neon (operands);
  72     default:
  73       gcc_unreachable ();
  74       return "";
  75     }
  76 }
  77   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store,mve_load")
  78    (set_attr "length" "4,8,8,4,8,8,4,4,4")
  79    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*,*")
  80    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*,*")])
  81
  82 (define_insn "*mve_vdup<mode>"
  83   [(set (match_operand:MVE_vecs 0 "s_register_operand" "=w")
  84         (vec_duplicate:MVE_vecs
  85           (match_operand:<V_elem> 1 "s_register_operand" "r")))]
  86   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
  87   "vdup.<V_sz_elem>\t%q0, %1"
  88   [(set_attr "length" "4")
  89    (set_attr "type" "mve_move")])
  90
  91 ;;
  92 ;; [vst4q])
  93 ;;
  94 (define_insn "mve_vst4q<mode>"
  95   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
  96         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
  97                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
  98          VST4Q))
  99   ]
 100   "TARGET_HAVE_MVE"
 101 {
 102    rtx ops[6];
 103    int regno = REGNO (operands[1]);
 104    ops[0] = gen_rtx_REG (TImode, regno);
 105    ops[1] = gen_rtx_REG (TImode, regno+4);
 106    ops[2] = gen_rtx_REG (TImode, regno+8);
 107    ops[3] = gen_rtx_REG (TImode, regno+12);
 108    rtx reg  = operands[0];
 109    while (reg && !REG_P (reg))
 110     reg = XEXP (reg, 0);
 111    gcc_assert (REG_P (reg));
 112    ops[4] = reg;
 113    ops[5] = operands[0];
 114    /* Here in first three instructions data is stored to ops[4]'s location but
 115       in the fourth instruction data is stored to operands[0], this is to
 116       support the writeback.  */
 117    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 118                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 119                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 120                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 121    return "";
 122 }
 123   [(set_attr "length" "16")])
 124
 125 ;;
 126 ;; [vrndq_m_f])
 127 ;;
 128 (define_insn "mve_vrndq_m_f<mode>"
 129   [
 130    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 131         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 132                        (match_operand:MVE_0 2 "s_register_operand" "w")
 133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 134          VRNDQ_M_F))
 135   ]
 136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 137   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 138   [(set_attr "type" "mve_move")
 139    (set_attr "length""8")])
 140
 141 ;;
 142 ;; [vrndxq_f])
 143 ;;
 144 (define_insn "mve_vrndxq_f<mode>"
 145   [
 146    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 147         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 148          VRNDXQ_F))
 149   ]
 150   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 151   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 152   [(set_attr "type" "mve_move")
 153 ])
 154
 155 ;;
 156 ;; [vrndq_f])
 157 ;;
 158 (define_insn "mve_vrndq_f<mode>"
 159   [
 160    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 161         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 162          VRNDQ_F))
 163   ]
 164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 165   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 166   [(set_attr "type" "mve_move")
 167 ])
 168
 169 ;;
 170 ;; [vrndpq_f])
 171 ;;
 172 (define_insn "mve_vrndpq_f<mode>"
 173   [
 174    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 175         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 176          VRNDPQ_F))
 177   ]
 178   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 179   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 180   [(set_attr "type" "mve_move")
 181 ])
 182
 183 ;;
 184 ;; [vrndnq_f])
 185 ;;
 186 (define_insn "mve_vrndnq_f<mode>"
 187   [
 188    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 189         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 190          VRNDNQ_F))
 191   ]
 192   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 193   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 194   [(set_attr "type" "mve_move")
 195 ])
 196
 197 ;;
 198 ;; [vrndmq_f])
 199 ;;
 200 (define_insn "mve_vrndmq_f<mode>"
 201   [
 202    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 203         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 204          VRNDMQ_F))
 205   ]
 206   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 207   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 208   [(set_attr "type" "mve_move")
 209 ])
 210
 211 ;;
 212 ;; [vrndaq_f])
 213 ;;
 214 (define_insn "mve_vrndaq_f<mode>"
 215   [
 216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 217         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 218          VRNDAQ_F))
 219   ]
 220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 221   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 222   [(set_attr "type" "mve_move")
 223 ])
 224
 225 ;;
 226 ;; [vrev64q_f])
 227 ;;
 228 (define_insn "mve_vrev64q_f<mode>"
 229   [
 230    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 232          VREV64Q_F))
 233   ]
 234   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 235   "vrev64.%#<V_sz_elem> %q0, %q1"
 236   [(set_attr "type" "mve_move")
 237 ])
 238
 239 ;;
 240 ;; [vnegq_f])
 241 ;;
 242 (define_insn "mve_vnegq_f<mode>"
 243   [
 244    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 245         (neg:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 246   ]
 247   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 248   "vneg.f%#<V_sz_elem>  %q0, %q1"
 249   [(set_attr "type" "mve_move")
 250 ])
 251
 252 ;;
 253 ;; [vdupq_n_f])
 254 ;;
 255 (define_insn "mve_vdupq_n_f<mode>"
 256   [
 257    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 258         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 259          VDUPQ_N_F))
 260   ]
 261   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 262   "vdup.%#<V_sz_elem>   %q0, %1"
 263   [(set_attr "type" "mve_move")
 264 ])
 265
 266 ;;
 267 ;; [vabsq_f])
 268 ;;
 269 (define_insn "mve_vabsq_f<mode>"
 270   [
 271    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 272         (abs:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")))
 273   ]
 274   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 275   "vabs.f%#<V_sz_elem>  %q0, %q1"
 276   [(set_attr "type" "mve_move")
 277 ])
 278
 279 ;;
 280 ;; [vrev32q_f])
 281 ;;
 282 (define_insn "mve_vrev32q_fv8hf"
 283   [
 284    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 285         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 286          VREV32Q_F))
 287   ]
 288   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 289   "vrev32.16 %q0, %q1"
 290   [(set_attr "type" "mve_move")
 291 ])
 292 ;;
 293 ;; [vcvttq_f32_f16])
 294 ;;
 295 (define_insn "mve_vcvttq_f32_f16v4sf"
 296   [
 297    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 298         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 299          VCVTTQ_F32_F16))
 300   ]
 301   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 302   "vcvtt.f32.f16 %q0, %q1"
 303   [(set_attr "type" "mve_move")
 304 ])
 305
 306 ;;
 307 ;; [vcvtbq_f32_f16])
 308 ;;
 309 (define_insn "mve_vcvtbq_f32_f16v4sf"
 310   [
 311    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 312         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 313          VCVTBQ_F32_F16))
 314   ]
 315   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 316   "vcvtb.f32.f16 %q0, %q1"
 317   [(set_attr "type" "mve_move")
 318 ])
 319
 320 ;;
 321 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 322 ;;
 323 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 324   [
 325    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 326         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 327          VCVTQ_TO_F))
 328   ]
 329   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 330   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 331   [(set_attr "type" "mve_move")
 332 ])
 333
 334 ;;
 335 ;; [vrev64q_u, vrev64q_s])
 336 ;;
 337 (define_insn "mve_vrev64q_<supf><mode>"
 338   [
 339    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 340         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 341          VREV64Q))
 342   ]
 343   "TARGET_HAVE_MVE"
 344   "vrev64.%#<V_sz_elem> %q0, %q1"
 345   [(set_attr "type" "mve_move")
 346 ])
 347
 348 ;;
 349 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 350 ;;
 351 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 352   [
 353    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 354         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 355          VCVTQ_FROM_F))
 356   ]
 357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 358   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
 359   [(set_attr "type" "mve_move")
 360 ])
 361 ;; [vqnegq_s])
 362 ;;
 363 (define_insn "mve_vqnegq_s<mode>"
 364   [
 365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 367          VQNEGQ_S))
 368   ]
 369   "TARGET_HAVE_MVE"
 370   "vqneg.s%#<V_sz_elem> %q0, %q1"
 371   [(set_attr "type" "mve_move")
 372 ])
 373
 374 ;;
 375 ;; [vqabsq_s])
 376 ;;
 377 (define_insn "mve_vqabsq_s<mode>"
 378   [
 379    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 380         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 381          VQABSQ_S))
 382   ]
 383   "TARGET_HAVE_MVE"
 384   "vqabs.s%#<V_sz_elem> %q0, %q1"
 385   [(set_attr "type" "mve_move")
 386 ])
 387
 388 ;;
 389 ;; [vnegq_s])
 390 ;;
 391 (define_insn "mve_vnegq_s<mode>"
 392   [
 393    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 394         (neg:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 395   ]
 396   "TARGET_HAVE_MVE"
 397   "vneg.s%#<V_sz_elem>  %q0, %q1"
 398   [(set_attr "type" "mve_move")
 399 ])
 400
 401 ;;
 402 ;; [vmvnq_u, vmvnq_s])
 403 ;;
 404 (define_insn "mve_vmvnq_u<mode>"
 405   [
 406    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 407         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 408   ]
 409   "TARGET_HAVE_MVE"
 410   "vmvn\t%q0, %q1"
 411   [(set_attr "type" "mve_move")
 412 ])
 413 (define_expand "mve_vmvnq_s<mode>"
 414   [
 415    (set (match_operand:MVE_2 0 "s_register_operand")
 416         (not:MVE_2 (match_operand:MVE_2 1 "s_register_operand")))
 417   ]
 418   "TARGET_HAVE_MVE"
 419 )
 420
 421 ;;
 422 ;; [vdupq_n_u, vdupq_n_s])
 423 ;;
 424 (define_insn "mve_vdupq_n_<supf><mode>"
 425   [
 426    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 427         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 428          VDUPQ_N))
 429   ]
 430   "TARGET_HAVE_MVE"
 431   "vdup.%#<V_sz_elem>   %q0, %1"
 432   [(set_attr "type" "mve_move")
 433 ])
 434
 435 ;;
 436 ;; [vclzq_u, vclzq_s])
 437 ;;
 438 (define_insn "mve_vclzq_<supf><mode>"
 439   [
 440    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 441         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 442          VCLZQ))
 443   ]
 444   "TARGET_HAVE_MVE"
 445   "vclz.i%#<V_sz_elem>  %q0, %q1"
 446   [(set_attr "type" "mve_move")
 447 ])
 448
 449 ;;
 450 ;; [vclsq_s])
 451 ;;
 452 (define_insn "mve_vclsq_s<mode>"
 453   [
 454    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 455         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 456          VCLSQ_S))
 457   ]
 458   "TARGET_HAVE_MVE"
 459   "vcls.s%#<V_sz_elem>  %q0, %q1"
 460   [(set_attr "type" "mve_move")
 461 ])
 462
 463 ;;
 464 ;; [vaddvq_s, vaddvq_u])
 465 ;;
 466 (define_insn "@mve_vaddvq_<supf><mode>"
 467   [
 468    (set (match_operand:SI 0 "s_register_operand" "=Te")
 469         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
 470          VADDVQ))
 471   ]
 472   "TARGET_HAVE_MVE"
 473   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
 474   [(set_attr "type" "mve_move")
 475 ])
 476
 477 ;;
 478 ;; [vabsq_s])
 479 ;;
 480 (define_insn "mve_vabsq_s<mode>"
 481   [
 482    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 483         (abs:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")))
 484   ]
 485   "TARGET_HAVE_MVE"
 486   "vabs.s%#<V_sz_elem>\t%q0, %q1"
 487   [(set_attr "type" "mve_move")
 488 ])
 489
 490 ;;
 491 ;; [vrev32q_u, vrev32q_s])
 492 ;;
 493 (define_insn "mve_vrev32q_<supf><mode>"
 494   [
 495    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
 496         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
 497          VREV32Q))
 498   ]
 499   "TARGET_HAVE_MVE"
 500   "vrev32.%#<V_sz_elem>\t%q0, %q1"
 501   [(set_attr "type" "mve_move")
 502 ])
 503
 504 ;;
 505 ;; [vmovltq_u, vmovltq_s])
 506 ;;
 507 (define_insn "mve_vmovltq_<supf><mode>"
 508   [
 509    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 510         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 511          VMOVLTQ))
 512   ]
 513   "TARGET_HAVE_MVE"
 514   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
 515   [(set_attr "type" "mve_move")
 516 ])
 517
 518 ;;
 519 ;; [vmovlbq_s, vmovlbq_u])
 520 ;;
 521 (define_insn "mve_vmovlbq_<supf><mode>"
 522   [
 523    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
 524         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
 525          VMOVLBQ))
 526   ]
 527   "TARGET_HAVE_MVE"
 528   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
 529   [(set_attr "type" "mve_move")
 530 ])
 531
 532 ;;
 533 ;; [vcvtpq_s, vcvtpq_u])
 534 ;;
 535 (define_insn "mve_vcvtpq_<supf><mode>"
 536   [
 537    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 538         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 539          VCVTPQ))
 540   ]
 541   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 542   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 543   [(set_attr "type" "mve_move")
 544 ])
 545
 546 ;;
 547 ;; [vcvtnq_s, vcvtnq_u])
 548 ;;
 549 (define_insn "mve_vcvtnq_<supf><mode>"
 550   [
 551    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 552         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 553          VCVTNQ))
 554   ]
 555   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 556   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 557   [(set_attr "type" "mve_move")
 558 ])
 559
 560 ;;
 561 ;; [vcvtmq_s, vcvtmq_u])
 562 ;;
 563 (define_insn "mve_vcvtmq_<supf><mode>"
 564   [
 565    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 566         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 567          VCVTMQ))
 568   ]
 569   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 570   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 571   [(set_attr "type" "mve_move")
 572 ])
 573
 574 ;;
 575 ;; [vcvtaq_u, vcvtaq_s])
 576 ;;
 577 (define_insn "mve_vcvtaq_<supf><mode>"
 578   [
 579    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 580         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 581          VCVTAQ))
 582   ]
 583   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 584   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
 585   [(set_attr "type" "mve_move")
 586 ])
 587
 588 ;;
 589 ;; [vmvnq_n_u, vmvnq_n_s])
 590 ;;
 591 (define_insn "mve_vmvnq_n_<supf><mode>"
 592   [
 593    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 594         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
 595          VMVNQ_N))
 596   ]
 597   "TARGET_HAVE_MVE"
 598   "vmvn.i%#<V_sz_elem>  %q0, %1"
 599   [(set_attr "type" "mve_move")
 600 ])
 601
 602 ;;
 603 ;; [vrev16q_u, vrev16q_s])
 604 ;;
 605 (define_insn "mve_vrev16q_<supf>v16qi"
 606   [
 607    (set (match_operand:V16QI 0 "s_register_operand" "=w")
 608         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
 609          VREV16Q))
 610   ]
 611   "TARGET_HAVE_MVE"
 612   "vrev16.8 %q0, %q1"
 613   [(set_attr "type" "mve_move")
 614 ])
 615
 616 ;;
 617 ;; [vaddlvq_s vaddlvq_u])
 618 ;;
 619 (define_insn "mve_vaddlvq_<supf>v4si"
 620   [
 621    (set (match_operand:DI 0 "s_register_operand" "=r")
 622         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
 623          VADDLVQ))
 624   ]
 625   "TARGET_HAVE_MVE"
 626   "vaddlv.<supf>32 %Q0, %R0, %q1"
 627   [(set_attr "type" "mve_move")
 628 ])
 629
 630 ;;
 631 ;; [vctp8q vctp16q vctp32q vctp64q])
 632 ;;
 633 (define_insn "mve_vctp<mode1>qhi"
 634   [
 635    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 636         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
 637         VCTPQ))
 638   ]
 639   "TARGET_HAVE_MVE"
 640   "vctp.<mode1> %1"
 641   [(set_attr "type" "mve_move")
 642 ])
 643
 644 ;;
 645 ;; [vpnot])
 646 ;;
 647 (define_insn "mve_vpnothi"
 648   [
 649    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 650         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
 651          VPNOT))
 652   ]
 653   "TARGET_HAVE_MVE"
 654   "vpnot"
 655   [(set_attr "type" "mve_move")
 656 ])
 657
 658 ;;
 659 ;; [vsubq_n_f])
 660 ;;
 661 (define_insn "mve_vsubq_n_f<mode>"
 662   [
 663    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 664         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 665                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 666          VSUBQ_N_F))
 667   ]
 668   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 669   "vsub.f<V_sz_elem>  %q0, %q1, %2"
 670   [(set_attr "type" "mve_move")
 671 ])
 672
 673 ;;
 674 ;; [vbrsrq_n_f])
 675 ;;
 676 (define_insn "mve_vbrsrq_n_f<mode>"
 677   [
 678    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 679         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
 680                        (match_operand:SI 2 "s_register_operand" "r")]
 681          VBRSRQ_N_F))
 682   ]
 683   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 684   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
 685   [(set_attr "type" "mve_move")
 686 ])
 687
 688 ;;
 689 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
 690 ;;
 691 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
 692   [
 693    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 694         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 695                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 696          VCVTQ_N_TO_F))
 697   ]
 698   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 699   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
 700   [(set_attr "type" "mve_move")
 701 ])
 702
 703 ;; [vcreateq_f])
 704 ;;
 705 (define_insn "mve_vcreateq_f<mode>"
 706   [
 707    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 708         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
 709                        (match_operand:DI 2 "s_register_operand" "r")]
 710          VCREATEQ_F))
 711   ]
 712   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 713   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 714   [(set_attr "type" "mve_move")
 715    (set_attr "length""8")])
 716
 717 ;;
 718 ;; [vcreateq_u, vcreateq_s])
 719 ;;
 720 (define_insn "mve_vcreateq_<supf><mode>"
 721   [
 722    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
 723         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
 724                        (match_operand:DI 2 "s_register_operand" "r")]
 725          VCREATEQ))
 726   ]
 727   "TARGET_HAVE_MVE"
 728   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
 729   [(set_attr "type" "mve_move")
 730    (set_attr "length""8")])
 731
 732 ;;
 733 ;; [vshrq_n_s, vshrq_n_u])
 734 ;;
 735 ;; Version that takes an immediate as operand 2.
 736 (define_insn "mve_vshrq_n_<supf><mode>"
 737   [
 738    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 739         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 740                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 741          VSHRQ_N))
 742   ]
 743   "TARGET_HAVE_MVE"
 744   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
 745   [(set_attr "type" "mve_move")
 746 ])
 747
 748 ;; Versions that take constant vectors as operand 2 (with all elements
 749 ;; equal).
 750 (define_insn "mve_vshrq_n_s<mode>_imm"
 751   [
 752    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 753         (ashiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 754                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 755   ]
 756   "TARGET_HAVE_MVE"
 757   {
 758     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 759                                         <MODE>mode,
 760                                         VALID_NEON_QREG_MODE (<MODE>mode),
 761                                         true);
 762   }
 763   [(set_attr "type" "mve_move")
 764 ])
 765 (define_insn "mve_vshrq_n_u<mode>_imm"
 766   [
 767    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 768         (lshiftrt:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
 769                         (match_operand:MVE_2 2 "imm_for_neon_rshift_operand" "i")))
 770   ]
 771   "TARGET_HAVE_MVE"
 772   {
 773     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 774                                         <MODE>mode,
 775                                         VALID_NEON_QREG_MODE (<MODE>mode),
 776                                         true);
 777   }
 778   [(set_attr "type" "mve_move")
 779 ])
 780
 781 ;;
 782 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
 783 ;;
 784 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
 785   [
 786    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 787         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
 788                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
 789          VCVTQ_N_FROM_F))
 790   ]
 791   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 792   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
 793   [(set_attr "type" "mve_move")
 794 ])
 795
 796 ;;
 797 ;; [vaddlvq_p_s])
 798 ;;
 799 (define_insn "mve_vaddlvq_p_<supf>v4si"
 800   [
 801    (set (match_operand:DI 0 "s_register_operand" "=r")
 802         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
 803                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 804          VADDLVQ_P))
 805   ]
 806   "TARGET_HAVE_MVE"
 807   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
 808   [(set_attr "type" "mve_move")
 809    (set_attr "length""8")])
 810
 811 ;;
 812 ;; [vcmpneq_, vcmpcsq_, vcmpeqq_, vcmpgeq_, vcmpgtq_, vcmphiq_, vcmpleq_, vcmpltq_])
 813 ;;
 814 (define_insn "@mve_vcmp<mve_cmp_op>q_<mode>"
 815   [
 816    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 817         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 818                     (match_operand:MVE_2 2 "s_register_operand" "w")))
 819   ]
 820   "TARGET_HAVE_MVE"
 821   "vcmp.<mve_cmp_type>%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
 822   [(set_attr "type" "mve_move")
 823 ])
 824
 825 ;;
 826 ;; [vcmpcsq_n_, vcmpeqq_n_, vcmpgeq_n_, vcmpgtq_n_, vcmphiq_n_, vcmpleq_n_, vcmpltq_n_, vcmpneq_n_])
 827 ;;
 828 (define_insn "mve_vcmp<mve_cmp_op>q_n_<mode>"
 829   [
 830    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
 831         (MVE_COMPARISONS:HI (match_operand:MVE_2 1 "s_register_operand" "w")
 832                     (match_operand:<V_elem> 2 "s_register_operand" "r")))
 833   ]
 834   "TARGET_HAVE_MVE"
 835   "vcmp.<mve_cmp_type>%#<V_sz_elem>     <mve_cmp_op>, %q1, %2"
 836   [(set_attr "type" "mve_move")
 837 ])
 838
 839 ;;
 840 ;; [vshlq_s, vshlq_u])
 841 ;; See vec-common.md
 842
 843 ;;
 844 ;; [vabdq_s, vabdq_u])
 845 ;;
 846 (define_insn "mve_vabdq_<supf><mode>"
 847   [
 848    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 849         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 850                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 851          VABDQ))
 852   ]
 853   "TARGET_HAVE_MVE"
 854   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
 855   [(set_attr "type" "mve_move")
 856 ])
 857
 858 ;;
 859 ;; [vaddq_n_s, vaddq_n_u])
 860 ;;
 861 (define_insn "mve_vaddq_n_<supf><mode>"
 862   [
 863    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 864         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 865                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
 866          VADDQ_N))
 867   ]
 868   "TARGET_HAVE_MVE"
 869   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
 870   [(set_attr "type" "mve_move")
 871 ])
 872
 873 ;;
 874 ;; [vaddvaq_s, vaddvaq_u])
 875 ;;
 876 (define_insn "mve_vaddvaq_<supf><mode>"
 877   [
 878    (set (match_operand:SI 0 "s_register_operand" "=Te")
 879         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
 880                     (match_operand:MVE_2 2 "s_register_operand" "w")]
 881          VADDVAQ))
 882   ]
 883   "TARGET_HAVE_MVE"
 884   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
 885   [(set_attr "type" "mve_move")
 886 ])
 887
 888 ;;
 889 ;; [vaddvq_p_u, vaddvq_p_s])
 890 ;;
 891 (define_insn "mve_vaddvq_p_<supf><mode>"
 892   [
 893    (set (match_operand:SI 0 "s_register_operand" "=Te")
 894         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
 895                     (match_operand:HI 2 "vpr_register_operand" "Up")]
 896          VADDVQ_P))
 897   ]
 898   "TARGET_HAVE_MVE"
 899   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
 900   [(set_attr "type" "mve_move")
 901    (set_attr "length""8")])
 902
 903 ;;
 904 ;; [vandq_u, vandq_s])
 905 ;;
 906 ;; signed and unsigned versions are the same: define the unsigned
 907 ;; insn, and use an expander for the signed one as we still reference
 908 ;; both names from arm_mve.h.
 909 ;; We use the same code as in neon.md (TODO: avoid this duplication).
 910 (define_insn "mve_vandq_u<mode>"
 911   [
 912    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
 913         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
 914                    (match_operand:MVE_2 2 "neon_inv_logic_op2" "w,DL")))
 915   ]
 916   "TARGET_HAVE_MVE"
 917   "@
 918    vand\t%q0, %q1, %q2
 919    * return neon_output_logic_immediate (\"vand\", &operands[2], <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));"
 920   [(set_attr "type" "mve_move")
 921 ])
 922 (define_expand "mve_vandq_s<mode>"
 923   [
 924    (set (match_operand:MVE_2 0 "s_register_operand")
 925         (and:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
 926                    (match_operand:MVE_2 2 "neon_inv_logic_op2")))
 927   ]
 928   "TARGET_HAVE_MVE"
 929 )
 930
 931 ;;
 932 ;; [vbicq_s, vbicq_u])
 933 ;;
 934 (define_insn "mve_vbicq_u<mode>"
 935   [
 936    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 937         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
 938                               (match_operand:MVE_2 1 "s_register_operand" "w")))
 939   ]
 940   "TARGET_HAVE_MVE"
 941   "vbic\t%q0, %q1, %q2"
 942   [(set_attr "type" "mve_move")
 943 ])
 944
 945 (define_expand "mve_vbicq_s<mode>"
 946   [
 947    (set (match_operand:MVE_2 0 "s_register_operand")
 948         (and:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
 949                    (match_operand:MVE_2 1 "s_register_operand")))
 950   ]
 951   "TARGET_HAVE_MVE"
 952 )
 953
 954 ;;
 955 ;; [vbrsrq_n_u, vbrsrq_n_s])
 956 ;;
 957 (define_insn "mve_vbrsrq_n_<supf><mode>"
 958   [
 959    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 960         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 961                        (match_operand:SI 2 "s_register_operand" "r")]
 962          VBRSRQ_N))
 963   ]
 964   "TARGET_HAVE_MVE"
 965   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
 966   [(set_attr "type" "mve_move")
 967 ])
 968
 969 ;;
 970 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
 971 ;;
 972 (define_insn "mve_vcaddq<mve_rot><mode>"
 973   [
 974    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
 975         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
 976                        (match_operand:MVE_2 2 "s_register_operand" "w")]
 977          VCADD))
 978   ]
 979   "TARGET_HAVE_MVE"
 980   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
 981   [(set_attr "type" "mve_move")
 982 ])
 983
 984 ;; Auto vectorizer pattern for int vcadd
 985 (define_expand "cadd<rot><mode>3"
 986   [(set (match_operand:MVE_2 0 "register_operand")
 987         (unspec:MVE_2 [(match_operand:MVE_2 1 "register_operand")
 988                        (match_operand:MVE_2 2 "register_operand")]
 989           VCADD))]
 990   "TARGET_HAVE_MVE && !BYTES_BIG_ENDIAN"
 991 )
 992
 993 ;;
 994 ;; [veorq_u, veorq_s])
 995 ;;
 996 (define_insn "mve_veorq_u<mode>"
 997   [
 998    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
 999         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1000                    (match_operand:MVE_2 2 "s_register_operand" "w")))
1001   ]
1002   "TARGET_HAVE_MVE"
1003   "veor\t%q0, %q1, %q2"
1004   [(set_attr "type" "mve_move")
1005 ])
1006 (define_expand "mve_veorq_s<mode>"
1007   [
1008    (set (match_operand:MVE_2 0 "s_register_operand")
1009         (xor:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1010                    (match_operand:MVE_2 2 "s_register_operand")))
1011   ]
1012   "TARGET_HAVE_MVE"
1013 )
1014
1015 ;;
1016 ;; [vhaddq_n_u, vhaddq_n_s])
1017 ;;
1018 (define_insn "mve_vhaddq_n_<supf><mode>"
1019   [
1020    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1021         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1022                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1023          VHADDQ_N))
1024   ]
1025   "TARGET_HAVE_MVE"
1026   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1027   [(set_attr "type" "mve_move")
1028 ])
1029
1030 ;;
1031 ;; [vhaddq_s, vhaddq_u])
1032 ;;
1033 (define_insn "@mve_vhaddq_<supf><mode>"
1034   [
1035    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1036         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1037                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1038          VHADDQ))
1039   ]
1040   "TARGET_HAVE_MVE"
1041   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1042   [(set_attr "type" "mve_move")
1043 ])
1044
1045 ;;
1046 ;; [vhcaddq_rot270_s])
1047 ;;
1048 (define_insn "mve_vhcaddq_rot270_s<mode>"
1049   [
1050    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1051         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1052                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1053          VHCADDQ_ROT270_S))
1054   ]
1055   "TARGET_HAVE_MVE"
1056   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1057   [(set_attr "type" "mve_move")
1058 ])
1059
1060 ;;
1061 ;; [vhcaddq_rot90_s])
1062 ;;
1063 (define_insn "mve_vhcaddq_rot90_s<mode>"
1064   [
1065    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1066         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1067                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1068          VHCADDQ_ROT90_S))
1069   ]
1070   "TARGET_HAVE_MVE"
1071   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1072   [(set_attr "type" "mve_move")
1073 ])
1074
1075 ;;
1076 ;; [vhsubq_n_u, vhsubq_n_s])
1077 ;;
1078 (define_insn "mve_vhsubq_n_<supf><mode>"
1079   [
1080    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1081         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1082                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1083          VHSUBQ_N))
1084   ]
1085   "TARGET_HAVE_MVE"
1086   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1087   [(set_attr "type" "mve_move")
1088 ])
1089
1090 ;;
1091 ;; [vhsubq_s, vhsubq_u])
1092 ;;
1093 (define_insn "mve_vhsubq_<supf><mode>"
1094   [
1095    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1096         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1097                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1098          VHSUBQ))
1099   ]
1100   "TARGET_HAVE_MVE"
1101   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1102   [(set_attr "type" "mve_move")
1103 ])
1104
1105 ;;
1106 ;; [vmaxaq_s])
1107 ;;
1108 (define_insn "mve_vmaxaq_s<mode>"
1109   [
1110    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1111         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1112                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1113          VMAXAQ_S))
1114   ]
1115   "TARGET_HAVE_MVE"
1116   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1117   [(set_attr "type" "mve_move")
1118 ])
1119
1120 ;;
1121 ;; [vmaxavq_s])
1122 ;;
1123 (define_insn "mve_vmaxavq_s<mode>"
1124   [
1125    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1126         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1127                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1128          VMAXAVQ_S))
1129   ]
1130   "TARGET_HAVE_MVE"
1131   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1132   [(set_attr "type" "mve_move")
1133 ])
1134
1135 ;;
1136 ;; [vmaxq_u, vmaxq_s])
1137 ;;
1138 (define_insn "mve_vmaxq_s<mode>"
1139   [
1140    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1141         (smax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1142                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1143   ]
1144   "TARGET_HAVE_MVE"
1145   "vmax.%#<V_s_elem>\t%q0, %q1, %q2"
1146   [(set_attr "type" "mve_move")
1147 ])
1148
1149 (define_insn "mve_vmaxq_u<mode>"
1150   [
1151    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1152         (umax:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1153                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1154   ]
1155   "TARGET_HAVE_MVE"
1156   "vmax.%#<V_u_elem>\t%q0, %q1, %q2"
1157   [(set_attr "type" "mve_move")
1158 ])
1159
1160 ;;
1161 ;; [vmaxvq_u, vmaxvq_s])
1162 ;;
1163 (define_insn "mve_vmaxvq_<supf><mode>"
1164   [
1165    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1166         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1167                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1168          VMAXVQ))
1169   ]
1170   "TARGET_HAVE_MVE"
1171   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1172   [(set_attr "type" "mve_move")
1173 ])
1174
1175 ;;
1176 ;; [vminaq_s])
1177 ;;
1178 (define_insn "mve_vminaq_s<mode>"
1179   [
1180    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1181         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1182                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1183          VMINAQ_S))
1184   ]
1185   "TARGET_HAVE_MVE"
1186   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1187   [(set_attr "type" "mve_move")
1188 ])
1189
1190 ;;
1191 ;; [vminavq_s])
1192 ;;
1193 (define_insn "mve_vminavq_s<mode>"
1194   [
1195    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1196         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1197                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1198          VMINAVQ_S))
1199   ]
1200   "TARGET_HAVE_MVE"
1201   "vminav.s%#<V_sz_elem>\t%0, %q2"
1202   [(set_attr "type" "mve_move")
1203 ])
1204
1205 ;;
1206 ;; [vminq_s, vminq_u])
1207 ;;
1208 (define_insn "mve_vminq_s<mode>"
1209   [
1210    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1211         (smin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1212                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1213   ]
1214   "TARGET_HAVE_MVE"
1215   "vmin.%#<V_s_elem>\t%q0, %q1, %q2"
1216   [(set_attr "type" "mve_move")
1217 ])
1218
1219 (define_insn "mve_vminq_u<mode>"
1220   [
1221    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1222         (umin:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1223                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1224   ]
1225   "TARGET_HAVE_MVE"
1226   "vmin.%#<V_u_elem>\t%q0, %q1, %q2"
1227   [(set_attr "type" "mve_move")
1228 ])
1229
1230 ;;
1231 ;; [vminvq_u, vminvq_s])
1232 ;;
1233 (define_insn "mve_vminvq_<supf><mode>"
1234   [
1235    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1236         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1237                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1238          VMINVQ))
1239   ]
1240   "TARGET_HAVE_MVE"
1241   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
1242   [(set_attr "type" "mve_move")
1243 ])
1244
1245 ;;
1246 ;; [vmladavq_u, vmladavq_s])
1247 ;;
1248 (define_insn "mve_vmladavq_<supf><mode>"
1249   [
1250    (set (match_operand:SI 0 "s_register_operand" "=Te")
1251         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1252                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1253          VMLADAVQ))
1254   ]
1255   "TARGET_HAVE_MVE"
1256   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
1257   [(set_attr "type" "mve_move")
1258 ])
1259
1260 ;;
1261 ;; [vmladavxq_s])
1262 ;;
1263 (define_insn "mve_vmladavxq_s<mode>"
1264   [
1265    (set (match_operand:SI 0 "s_register_operand" "=Te")
1266         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1267                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1268          VMLADAVXQ_S))
1269   ]
1270   "TARGET_HAVE_MVE"
1271   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1272   [(set_attr "type" "mve_move")
1273 ])
1274
1275 ;;
1276 ;; [vmlsdavq_s])
1277 ;;
1278 (define_insn "mve_vmlsdavq_s<mode>"
1279   [
1280    (set (match_operand:SI 0 "s_register_operand" "=Te")
1281         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1282                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1283          VMLSDAVQ_S))
1284   ]
1285   "TARGET_HAVE_MVE"
1286   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
1287   [(set_attr "type" "mve_move")
1288 ])
1289
1290 ;;
1291 ;; [vmlsdavxq_s])
1292 ;;
1293 (define_insn "mve_vmlsdavxq_s<mode>"
1294   [
1295    (set (match_operand:SI 0 "s_register_operand" "=Te")
1296         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1297                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1298          VMLSDAVXQ_S))
1299   ]
1300   "TARGET_HAVE_MVE"
1301   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
1302   [(set_attr "type" "mve_move")
1303 ])
1304
1305 ;;
1306 ;; [vmulhq_s, vmulhq_u])
1307 ;;
1308 (define_insn "mve_vmulhq_<supf><mode>"
1309   [
1310    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1311         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1312                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1313          VMULHQ))
1314   ]
1315   "TARGET_HAVE_MVE"
1316   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1317   [(set_attr "type" "mve_move")
1318 ])
1319
1320 ;;
1321 ;; [vmullbq_int_u, vmullbq_int_s])
1322 ;;
1323 (define_insn "mve_vmullbq_int_<supf><mode>"
1324   [
1325    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1326         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1327                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1328          VMULLBQ_INT))
1329   ]
1330   "TARGET_HAVE_MVE"
1331   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1332   [(set_attr "type" "mve_move")
1333 ])
1334
1335 ;;
1336 ;; [vmulltq_int_u, vmulltq_int_s])
1337 ;;
1338 (define_insn "mve_vmulltq_int_<supf><mode>"
1339   [
1340    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
1341         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
1342                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
1343          VMULLTQ_INT))
1344   ]
1345   "TARGET_HAVE_MVE"
1346   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1347   [(set_attr "type" "mve_move")
1348 ])
1349
1350 ;;
1351 ;; [vmulq_n_u, vmulq_n_s])
1352 ;;
1353 (define_insn "mve_vmulq_n_<supf><mode>"
1354   [
1355    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1356         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1357                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1358          VMULQ_N))
1359   ]
1360   "TARGET_HAVE_MVE"
1361   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
1362   [(set_attr "type" "mve_move")
1363 ])
1364
1365 ;;
1366 ;; [vmulq_u, vmulq_s])
1367 ;;
1368 (define_insn "mve_vmulq_<supf><mode>"
1369   [
1370    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1371         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1372                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1373          VMULQ))
1374   ]
1375   "TARGET_HAVE_MVE"
1376   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1377   [(set_attr "type" "mve_move")
1378 ])
1379
1380 (define_insn "mve_vmulq<mode>"
1381   [
1382    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1383         (mult:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1384                     (match_operand:MVE_2 2 "s_register_operand" "w")))
1385   ]
1386   "TARGET_HAVE_MVE"
1387   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
1388   [(set_attr "type" "mve_move")
1389 ])
1390
1391 ;;
1392 ;; [vornq_u, vornq_s])
1393 ;;
1394 (define_insn "mve_vornq_s<mode>"
1395   [
1396    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1397         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand" "w"))
1398                    (match_operand:MVE_2 1 "s_register_operand" "w")))
1399   ]
1400   "TARGET_HAVE_MVE"
1401    "vorn\t%q0, %q1, %q2"
1402   [(set_attr "type" "mve_move")
1403 ])
1404
1405 (define_expand "mve_vornq_u<mode>"
1406   [
1407    (set (match_operand:MVE_2 0 "s_register_operand")
1408         (ior:MVE_2 (not:MVE_2 (match_operand:MVE_2 2 "s_register_operand"))
1409                    (match_operand:MVE_2 1 "s_register_operand")))
1410   ]
1411   "TARGET_HAVE_MVE"
1412 )
1413
1414 ;;
1415 ;; [vorrq_s, vorrq_u])
1416 ;;
1417 ;; signed and unsigned versions are the same: define the unsigned
1418 ;; insn, and use an expander for the signed one as we still reference
1419 ;; both names from arm_mve.h.
1420 ;; We use the same code as in neon.md (TODO: avoid this duplication).
1421 (define_insn "mve_vorrq_s<mode>"
1422   [
1423    (set (match_operand:MVE_2 0 "s_register_operand" "=w,w")
1424         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w,0")
1425                    (match_operand:MVE_2 2 "neon_logic_op2" "w,Dl")))
1426   ]
1427   "TARGET_HAVE_MVE"
1428   "@
1429    vorr\t%q0, %q1, %q2
1430    * return neon_output_logic_immediate (\"vorr\", &operands[2], <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));"
1431   [(set_attr "type" "mve_move")
1432 ])
1433 (define_expand "mve_vorrq_u<mode>"
1434   [
1435    (set (match_operand:MVE_2 0 "s_register_operand")
1436         (ior:MVE_2 (match_operand:MVE_2 1 "s_register_operand")
1437                    (match_operand:MVE_2 2 "neon_logic_op2")))
1438   ]
1439   "TARGET_HAVE_MVE"
1440 )
1441
1442 ;;
1443 ;; [vqaddq_n_s, vqaddq_n_u])
1444 ;;
1445 (define_insn "mve_vqaddq_n_<supf><mode>"
1446   [
1447    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1448         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1449                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1450          VQADDQ_N))
1451   ]
1452   "TARGET_HAVE_MVE"
1453   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1454   [(set_attr "type" "mve_move")
1455 ])
1456
1457 ;;
1458 ;; [vqaddq_u, vqaddq_s])
1459 ;;
1460 (define_insn "mve_vqaddq_<supf><mode>"
1461   [
1462    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1463         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1464                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1465          VQADDQ))
1466   ]
1467   "TARGET_HAVE_MVE"
1468   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1469   [(set_attr "type" "mve_move")
1470 ])
1471
1472 ;;
1473 ;; [vqdmulhq_n_s])
1474 ;;
1475 (define_insn "mve_vqdmulhq_n_s<mode>"
1476   [
1477    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1478         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1479                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1480          VQDMULHQ_N_S))
1481   ]
1482   "TARGET_HAVE_MVE"
1483   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1484   [(set_attr "type" "mve_move")
1485 ])
1486
1487 ;;
1488 ;; [vqdmulhq_s])
1489 ;;
1490 (define_insn "mve_vqdmulhq_s<mode>"
1491   [
1492    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1493         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1494                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1495          VQDMULHQ_S))
1496   ]
1497   "TARGET_HAVE_MVE"
1498   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1499   [(set_attr "type" "mve_move")
1500 ])
1501
1502 ;;
1503 ;; [vqrdmulhq_n_s])
1504 ;;
1505 (define_insn "mve_vqrdmulhq_n_s<mode>"
1506   [
1507    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1508         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1509                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1510          VQRDMULHQ_N_S))
1511   ]
1512   "TARGET_HAVE_MVE"
1513   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
1514   [(set_attr "type" "mve_move")
1515 ])
1516
1517 ;;
1518 ;; [vqrdmulhq_s])
1519 ;;
1520 (define_insn "mve_vqrdmulhq_s<mode>"
1521   [
1522    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1523         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1524                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1525          VQRDMULHQ_S))
1526   ]
1527   "TARGET_HAVE_MVE"
1528   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
1529   [(set_attr "type" "mve_move")
1530 ])
1531
1532 ;;
1533 ;; [vqrshlq_n_s, vqrshlq_n_u])
1534 ;;
1535 (define_insn "mve_vqrshlq_n_<supf><mode>"
1536   [
1537    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1538         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1539                        (match_operand:SI 2 "s_register_operand" "r")]
1540          VQRSHLQ_N))
1541   ]
1542   "TARGET_HAVE_MVE"
1543   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1544   [(set_attr "type" "mve_move")
1545 ])
1546
1547 ;;
1548 ;; [vqrshlq_s, vqrshlq_u])
1549 ;;
1550 (define_insn "mve_vqrshlq_<supf><mode>"
1551   [
1552    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1553         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1554                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1555          VQRSHLQ))
1556   ]
1557   "TARGET_HAVE_MVE"
1558   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1559   [(set_attr "type" "mve_move")
1560 ])
1561
1562 ;;
1563 ;; [vqshlq_n_s, vqshlq_n_u])
1564 ;;
1565 (define_insn "mve_vqshlq_n_<supf><mode>"
1566   [
1567    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1568         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1569                        (match_operand:SI 2 "immediate_operand" "i")]
1570          VQSHLQ_N))
1571   ]
1572   "TARGET_HAVE_MVE"
1573   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1574   [(set_attr "type" "mve_move")
1575 ])
1576
1577 ;;
1578 ;; [vqshlq_r_u, vqshlq_r_s])
1579 ;;
1580 (define_insn "mve_vqshlq_r_<supf><mode>"
1581   [
1582    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1583         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1584                        (match_operand:SI 2 "s_register_operand" "r")]
1585          VQSHLQ_R))
1586   ]
1587   "TARGET_HAVE_MVE"
1588   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
1589   [(set_attr "type" "mve_move")
1590 ])
1591
1592 ;;
1593 ;; [vqshlq_s, vqshlq_u])
1594 ;;
1595 (define_insn "mve_vqshlq_<supf><mode>"
1596   [
1597    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1598         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1599                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1600          VQSHLQ))
1601   ]
1602   "TARGET_HAVE_MVE"
1603   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1604   [(set_attr "type" "mve_move")
1605 ])
1606
1607 ;;
1608 ;; [vqshluq_n_s])
1609 ;;
1610 (define_insn "mve_vqshluq_n_s<mode>"
1611   [
1612    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1613         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1614                        (match_operand:SI 2 "mve_imm_7" "Ra")]
1615          VQSHLUQ_N_S))
1616   ]
1617   "TARGET_HAVE_MVE"
1618   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
1619   [(set_attr "type" "mve_move")
1620 ])
1621
1622 ;;
1623 ;; [vqsubq_n_s, vqsubq_n_u])
1624 ;;
1625 (define_insn "mve_vqsubq_n_<supf><mode>"
1626   [
1627    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1628         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1629                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1630          VQSUBQ_N))
1631   ]
1632   "TARGET_HAVE_MVE"
1633   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1634   [(set_attr "type" "mve_move")
1635 ])
1636
1637 ;;
1638 ;; [vqsubq_u, vqsubq_s])
1639 ;;
1640 (define_insn "mve_vqsubq_<supf><mode>"
1641   [
1642    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1643         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1644                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1645          VQSUBQ))
1646   ]
1647   "TARGET_HAVE_MVE"
1648   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1649   [(set_attr "type" "mve_move")
1650 ])
1651
1652 ;;
1653 ;; [vrhaddq_s, vrhaddq_u])
1654 ;;
1655 (define_insn "@mve_vrhaddq_<supf><mode>"
1656   [
1657    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1658         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1659                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1660          VRHADDQ))
1661   ]
1662   "TARGET_HAVE_MVE"
1663   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1664   [(set_attr "type" "mve_move")
1665 ])
1666
1667 ;;
1668 ;; [vrmulhq_s, vrmulhq_u])
1669 ;;
1670 (define_insn "mve_vrmulhq_<supf><mode>"
1671   [
1672    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1673         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1674                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1675          VRMULHQ))
1676   ]
1677   "TARGET_HAVE_MVE"
1678   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1679   [(set_attr "type" "mve_move")
1680 ])
1681
1682 ;;
1683 ;; [vrshlq_n_u, vrshlq_n_s])
1684 ;;
1685 (define_insn "mve_vrshlq_n_<supf><mode>"
1686   [
1687    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1688         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1689                        (match_operand:SI 2 "s_register_operand" "r")]
1690          VRSHLQ_N))
1691   ]
1692   "TARGET_HAVE_MVE"
1693   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
1694   [(set_attr "type" "mve_move")
1695 ])
1696
1697 ;;
1698 ;; [vrshlq_s, vrshlq_u])
1699 ;;
1700 (define_insn "mve_vrshlq_<supf><mode>"
1701   [
1702    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1703         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1704                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1705          VRSHLQ))
1706   ]
1707   "TARGET_HAVE_MVE"
1708   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1709   [(set_attr "type" "mve_move")
1710 ])
1711
1712 ;;
1713 ;; [vrshrq_n_s, vrshrq_n_u])
1714 ;;
1715 (define_insn "mve_vrshrq_n_<supf><mode>"
1716   [
1717    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1718         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1719                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1720          VRSHRQ_N))
1721   ]
1722   "TARGET_HAVE_MVE"
1723   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1724   [(set_attr "type" "mve_move")
1725 ])
1726
1727 ;;
1728 ;; [vshlq_n_u, vshlq_n_s])
1729 ;;
1730 (define_insn "mve_vshlq_n_<supf><mode>"
1731   [
1732    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1733         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1734                        (match_operand:SI 2 "immediate_operand" "i")]
1735          VSHLQ_N))
1736   ]
1737   "TARGET_HAVE_MVE"
1738   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1739   [(set_attr "type" "mve_move")
1740 ])
1741
1742 ;;
1743 ;; [vshlq_r_s, vshlq_r_u])
1744 ;;
1745 (define_insn "mve_vshlq_r_<supf><mode>"
1746   [
1747    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1748         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1749                        (match_operand:SI 2 "s_register_operand" "r")]
1750          VSHLQ_R))
1751   ]
1752   "TARGET_HAVE_MVE"
1753   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
1754   [(set_attr "type" "mve_move")
1755 ])
1756
1757 ;;
1758 ;; [vsubq_n_s, vsubq_n_u])
1759 ;;
1760 (define_insn "mve_vsubq_n_<supf><mode>"
1761   [
1762    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1763         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1764                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1765          VSUBQ_N))
1766   ]
1767   "TARGET_HAVE_MVE"
1768   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
1769   [(set_attr "type" "mve_move")
1770 ])
1771
1772 ;;
1773 ;; [vsubq_s, vsubq_u])
1774 ;;
1775 (define_insn "mve_vsubq_<supf><mode>"
1776   [
1777    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1778         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1779                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1780          VSUBQ))
1781   ]
1782   "TARGET_HAVE_MVE"
1783   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1784   [(set_attr "type" "mve_move")
1785 ])
1786
1787 (define_insn "mve_vsubq<mode>"
1788   [
1789    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1790         (minus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
1791                      (match_operand:MVE_2 2 "s_register_operand" "w")))
1792   ]
1793   "TARGET_HAVE_MVE"
1794   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
1795   [(set_attr "type" "mve_move")
1796 ])
1797
1798 ;;
1799 ;; [vabdq_f])
1800 ;;
1801 (define_insn "mve_vabdq_f<mode>"
1802   [
1803    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1804         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1805                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1806          VABDQ_F))
1807   ]
1808   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1809   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
1810   [(set_attr "type" "mve_move")
1811 ])
1812
1813 ;;
1814 ;; [vaddlvaq_s vaddlvaq_u])
1815 ;;
1816 (define_insn "mve_vaddlvaq_<supf>v4si"
1817   [
1818    (set (match_operand:DI 0 "s_register_operand" "=r")
1819         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
1820                     (match_operand:V4SI 2 "s_register_operand" "w")]
1821          VADDLVAQ))
1822   ]
1823   "TARGET_HAVE_MVE"
1824   "vaddlva.<supf>32 %Q0, %R0, %q2"
1825   [(set_attr "type" "mve_move")
1826 ])
1827
1828 ;;
1829 ;; [vaddq_n_f])
1830 ;;
1831 (define_insn "mve_vaddq_n_f<mode>"
1832   [
1833    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1834         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1835                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1836          VADDQ_N_F))
1837   ]
1838   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1839   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
1840   [(set_attr "type" "mve_move")
1841 ])
1842
1843 ;;
1844 ;; [vandq_f])
1845 ;;
1846 (define_insn "mve_vandq_f<mode>"
1847   [
1848    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1849         (and:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1850                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1851   ]
1852   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1853   "vand %q0, %q1, %q2"
1854   [(set_attr "type" "mve_move")
1855 ])
1856
1857 ;;
1858 ;; [vbicq_f])
1859 ;;
1860 (define_insn "mve_vbicq_f<mode>"
1861   [
1862    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1863         (and:MVE_0 (not:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w"))
1864                               (match_operand:MVE_0 2 "s_register_operand" "w")))
1865   ]
1866   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1867   "vbic %q0, %q1, %q2"
1868   [(set_attr "type" "mve_move")
1869 ])
1870
1871 ;;
1872 ;; [vbicq_n_s, vbicq_n_u])
1873 ;;
1874 (define_insn "mve_vbicq_n_<supf><mode>"
1875   [
1876    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1877         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
1878                        (match_operand:SI 2 "immediate_operand" "i")]
1879          VBICQ_N))
1880   ]
1881   "TARGET_HAVE_MVE"
1882   "vbic.i%#<V_sz_elem>  %q0, %2"
1883   [(set_attr "type" "mve_move")
1884 ])
1885
1886 ;;
1887 ;; [vcaddq, vcaddq_rot90, vcadd_rot180, vcadd_rot270])
1888 ;;
1889 (define_insn "mve_vcaddq<mve_rot><mode>"
1890   [
1891    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1892         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1893                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1894          VCADD))
1895   ]
1896   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1897   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1898   [(set_attr "type" "mve_move")
1899 ])
1900
1901 ;;
1902 ;; [vcmpeqq_f, vcmpgeq_f, vcmpgtq_f, vcmpleq_f, vcmpltq_f, vcmpneq_f])
1903 ;;
1904 (define_insn "@mve_vcmp<mve_cmp_op>q_f<mode>"
1905   [
1906    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1907         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1908                                (match_operand:MVE_0 2 "s_register_operand" "w")))
1909   ]
1910   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1911   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %q2"
1912   [(set_attr "type" "mve_move")
1913 ])
1914
1915 ;;
1916 ;; [vcmpeqq_n_f, vcmpgeq_n_f, vcmpgtq_n_f, vcmpleq_n_f, vcmpltq_n_f, vcmpneq_n_f])
1917 ;;
1918 (define_insn "@mve_vcmp<mve_cmp_op>q_n_f<mode>"
1919   [
1920    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1921         (MVE_FP_COMPARISONS:HI (match_operand:MVE_0 1 "s_register_operand" "w")
1922                                (match_operand:<V_elem> 2 "s_register_operand" "r")))
1923   ]
1924   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1925   "vcmp.f%#<V_sz_elem>  <mve_cmp_op>, %q1, %2"
1926   [(set_attr "type" "mve_move")
1927 ])
1928
1929 ;;
1930 ;; [vcmulq, vcmulq_rot90, vcmulq_rot180, vcmulq_rot270])
1931 ;;
1932 (define_insn "mve_vcmulq<mve_rot><mode>"
1933   [
1934    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
1935         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1936                        (match_operand:MVE_0 2 "s_register_operand" "w")]
1937          VCMUL))
1938   ]
1939   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1940   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #<rot>"
1941   [(set_attr "type" "mve_move")
1942 ])
1943
1944 ;;
1945 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
1946 ;;
1947 (define_insn "mve_vctp<mode1>q_mhi"
1948   [
1949    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1950         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
1951                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1952          VCTPQ_M))
1953   ]
1954   "TARGET_HAVE_MVE"
1955   "vpst\;vctpt.<mode1> %1"
1956   [(set_attr "type" "mve_move")
1957    (set_attr "length""8")])
1958
1959 ;;
1960 ;; [vcvtbq_f16_f32])
1961 ;;
1962 (define_insn "mve_vcvtbq_f16_f32v8hf"
1963   [
1964    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1965         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1966                       (match_operand:V4SF 2 "s_register_operand" "w")]
1967          VCVTBQ_F16_F32))
1968   ]
1969   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1970   "vcvtb.f16.f32 %q0, %q2"
1971   [(set_attr "type" "mve_move")
1972 ])
1973
1974 ;;
1975 ;; [vcvttq_f16_f32])
1976 ;;
1977 (define_insn "mve_vcvttq_f16_f32v8hf"
1978   [
1979    (set (match_operand:V8HF 0 "s_register_operand" "=w")
1980         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
1981                       (match_operand:V4SF 2 "s_register_operand" "w")]
1982          VCVTTQ_F16_F32))
1983   ]
1984   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1985   "vcvtt.f16.f32 %q0, %q2"
1986   [(set_attr "type" "mve_move")
1987 ])
1988
1989 ;;
1990 ;; [veorq_f])
1991 ;;
1992 (define_insn "mve_veorq_f<mode>"
1993   [
1994    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1995         (xor:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
1996                    (match_operand:MVE_0 2 "s_register_operand" "w")))
1997   ]
1998   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1999   "veor %q0, %q1, %q2"
2000   [(set_attr "type" "mve_move")
2001 ])
2002
2003 ;;
2004 ;; [vmaxnmaq_f])
2005 ;;
2006 (define_insn "mve_vmaxnmaq_f<mode>"
2007   [
2008    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2009         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2010                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2011          VMAXNMAQ_F))
2012   ]
2013   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2014   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2015   [(set_attr "type" "mve_move")
2016 ])
2017
2018 ;;
2019 ;; [vmaxnmavq_f])
2020 ;;
2021 (define_insn "mve_vmaxnmavq_f<mode>"
2022   [
2023    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2024         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2025                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2026          VMAXNMAVQ_F))
2027   ]
2028   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2029   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2030   [(set_attr "type" "mve_move")
2031 ])
2032
2033 ;;
2034 ;; [vmaxnmq_f])
2035 ;;
2036 (define_insn "mve_vmaxnmq_f<mode>"
2037   [
2038    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2039         (smax:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2040                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2041   ]
2042   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2043   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2044   [(set_attr "type" "mve_move")
2045 ])
2046
2047 ;;
2048 ;; [vmaxnmvq_f])
2049 ;;
2050 (define_insn "mve_vmaxnmvq_f<mode>"
2051   [
2052    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2053         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2054                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2055          VMAXNMVQ_F))
2056   ]
2057   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2058   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
2059   [(set_attr "type" "mve_move")
2060 ])
2061
2062 ;;
2063 ;; [vminnmaq_f])
2064 ;;
2065 (define_insn "mve_vminnmaq_f<mode>"
2066   [
2067    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2068         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2069                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2070          VMINNMAQ_F))
2071   ]
2072   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2073   "vminnma.f%#<V_sz_elem>       %q0, %q2"
2074   [(set_attr "type" "mve_move")
2075 ])
2076
2077 ;;
2078 ;; [vminnmavq_f])
2079 ;;
2080 (define_insn "mve_vminnmavq_f<mode>"
2081   [
2082    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2083         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2084                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2085          VMINNMAVQ_F))
2086   ]
2087   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2088   "vminnmav.f%#<V_sz_elem>      %0, %q2"
2089   [(set_attr "type" "mve_move")
2090 ])
2091
2092 ;;
2093 ;; [vminnmq_f])
2094 ;;
2095 (define_insn "mve_vminnmq_f<mode>"
2096   [
2097    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2098         (smin:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2099                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2100   ]
2101   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2102   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
2103   [(set_attr "type" "mve_move")
2104 ])
2105
2106 ;;
2107 ;; [vminnmvq_f])
2108 ;;
2109 (define_insn "mve_vminnmvq_f<mode>"
2110   [
2111    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2112         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2113                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2114          VMINNMVQ_F))
2115   ]
2116   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2117   "vminnmv.f%#<V_sz_elem>       %0, %q2"
2118   [(set_attr "type" "mve_move")
2119 ])
2120
2121 ;;
2122 ;; [vmlaldavq_u, vmlaldavq_s])
2123 ;;
2124 (define_insn "mve_vmlaldavq_<supf><mode>"
2125   [
2126    (set (match_operand:DI 0 "s_register_operand" "=r")
2127         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2128                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2129          VMLALDAVQ))
2130   ]
2131   "TARGET_HAVE_MVE"
2132   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2133   [(set_attr "type" "mve_move")
2134 ])
2135
2136 ;;
2137 ;; [vmlaldavxq_s])
2138 ;;
2139 (define_insn "mve_vmlaldavxq_s<mode>"
2140   [
2141    (set (match_operand:DI 0 "s_register_operand" "=r")
2142         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2143                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2144          VMLALDAVXQ_S))
2145   ]
2146   "TARGET_HAVE_MVE"
2147   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2148   [(set_attr "type" "mve_move")
2149 ])
2150
2151 ;;
2152 ;; [vmlsldavq_s])
2153 ;;
2154 (define_insn "mve_vmlsldavq_s<mode>"
2155   [
2156    (set (match_operand:DI 0 "s_register_operand" "=r")
2157         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2158                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2159          VMLSLDAVQ_S))
2160   ]
2161   "TARGET_HAVE_MVE"
2162   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2163   [(set_attr "type" "mve_move")
2164 ])
2165
2166 ;;
2167 ;; [vmlsldavxq_s])
2168 ;;
2169 (define_insn "mve_vmlsldavxq_s<mode>"
2170   [
2171    (set (match_operand:DI 0 "s_register_operand" "=r")
2172         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
2173                     (match_operand:MVE_5 2 "s_register_operand" "w")]
2174          VMLSLDAVXQ_S))
2175   ]
2176   "TARGET_HAVE_MVE"
2177   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
2178   [(set_attr "type" "mve_move")
2179 ])
2180
2181 ;;
2182 ;; [vmovnbq_u, vmovnbq_s])
2183 ;;
2184 (define_insn "mve_vmovnbq_<supf><mode>"
2185   [
2186    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2187         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2188                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2189          VMOVNBQ))
2190   ]
2191   "TARGET_HAVE_MVE"
2192   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
2193   [(set_attr "type" "mve_move")
2194 ])
2195
2196 ;;
2197 ;; [vmovntq_s, vmovntq_u])
2198 ;;
2199 (define_insn "mve_vmovntq_<supf><mode>"
2200   [
2201    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2202         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2203                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2204          VMOVNTQ))
2205   ]
2206   "TARGET_HAVE_MVE"
2207   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
2208   [(set_attr "type" "mve_move")
2209 ])
2210
2211 ;;
2212 ;; [vmulq_f])
2213 ;;
2214 (define_insn "mve_vmulq_f<mode>"
2215   [
2216    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2217         (mult:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2218                     (match_operand:MVE_0 2 "s_register_operand" "w")))
2219   ]
2220   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2221   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
2222   [(set_attr "type" "mve_move")
2223 ])
2224
2225 ;;
2226 ;; [vmulq_n_f])
2227 ;;
2228 (define_insn "mve_vmulq_n_f<mode>"
2229   [
2230    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2231         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2232                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2233          VMULQ_N_F))
2234   ]
2235   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2236   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
2237   [(set_attr "type" "mve_move")
2238 ])
2239
2240 ;;
2241 ;; [vornq_f])
2242 ;;
2243 (define_insn "mve_vornq_f<mode>"
2244   [
2245    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2246         (ior:MVE_0 (not:MVE_0 (match_operand:MVE_0 2 "s_register_operand" "w"))
2247                    (match_operand:MVE_0 1 "s_register_operand" "w")))
2248   ]
2249   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2250   "vorn %q0, %q1, %q2"
2251   [(set_attr "type" "mve_move")
2252 ])
2253
2254 ;;
2255 ;; [vorrq_f])
2256 ;;
2257 (define_insn "mve_vorrq_f<mode>"
2258   [
2259    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2260         (ior:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2261                    (match_operand:MVE_0 2 "s_register_operand" "w")))
2262   ]
2263   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2264   "vorr %q0, %q1, %q2"
2265   [(set_attr "type" "mve_move")
2266 ])
2267
2268 ;;
2269 ;; [vorrq_n_u, vorrq_n_s])
2270 ;;
2271 (define_insn "mve_vorrq_n_<supf><mode>"
2272   [
2273    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2274         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2275                        (match_operand:SI 2 "immediate_operand" "i")]
2276          VORRQ_N))
2277   ]
2278   "TARGET_HAVE_MVE"
2279   "vorr.i%#<V_sz_elem>  %q0, %2"
2280   [(set_attr "type" "mve_move")
2281 ])
2282
2283 ;;
2284 ;; [vqdmullbq_n_s])
2285 ;;
2286 (define_insn "mve_vqdmullbq_n_s<mode>"
2287   [
2288    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2289         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2290                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2291          VQDMULLBQ_N_S))
2292   ]
2293   "TARGET_HAVE_MVE"
2294   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
2295   [(set_attr "type" "mve_move")
2296 ])
2297
2298 ;;
2299 ;; [vqdmullbq_s])
2300 ;;
2301 (define_insn "mve_vqdmullbq_s<mode>"
2302   [
2303    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2304         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2305                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2306          VQDMULLBQ_S))
2307   ]
2308   "TARGET_HAVE_MVE"
2309   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
2310   [(set_attr "type" "mve_move")
2311 ])
2312
2313 ;;
2314 ;; [vqdmulltq_n_s])
2315 ;;
2316 (define_insn "mve_vqdmulltq_n_s<mode>"
2317   [
2318    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2319         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2320                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
2321          VQDMULLTQ_N_S))
2322   ]
2323   "TARGET_HAVE_MVE"
2324   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
2325   [(set_attr "type" "mve_move")
2326 ])
2327
2328 ;;
2329 ;; [vqdmulltq_s])
2330 ;;
2331 (define_insn "mve_vqdmulltq_s<mode>"
2332   [
2333    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2334         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
2335                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
2336          VQDMULLTQ_S))
2337   ]
2338   "TARGET_HAVE_MVE"
2339   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
2340   [(set_attr "type" "mve_move")
2341 ])
2342
2343 ;;
2344 ;; [vqmovnbq_u, vqmovnbq_s])
2345 ;;
2346 (define_insn "mve_vqmovnbq_<supf><mode>"
2347   [
2348    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2349         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2350                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2351          VQMOVNBQ))
2352   ]
2353   "TARGET_HAVE_MVE"
2354   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
2355   [(set_attr "type" "mve_move")
2356 ])
2357
2358 ;;
2359 ;; [vqmovntq_u, vqmovntq_s])
2360 ;;
2361 (define_insn "mve_vqmovntq_<supf><mode>"
2362   [
2363    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2364         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2365                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2366          VQMOVNTQ))
2367   ]
2368   "TARGET_HAVE_MVE"
2369   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
2370   [(set_attr "type" "mve_move")
2371 ])
2372
2373 ;;
2374 ;; [vqmovunbq_s])
2375 ;;
2376 (define_insn "mve_vqmovunbq_s<mode>"
2377   [
2378    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2379         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2380                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2381          VQMOVUNBQ_S))
2382   ]
2383   "TARGET_HAVE_MVE"
2384   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
2385   [(set_attr "type" "mve_move")
2386 ])
2387
2388 ;;
2389 ;; [vqmovuntq_s])
2390 ;;
2391 (define_insn "mve_vqmovuntq_s<mode>"
2392   [
2393    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2394         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2395                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
2396          VQMOVUNTQ_S))
2397   ]
2398   "TARGET_HAVE_MVE"
2399   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
2400   [(set_attr "type" "mve_move")
2401 ])
2402
2403 ;;
2404 ;; [vrmlaldavhxq_s])
2405 ;;
2406 (define_insn "mve_vrmlaldavhxq_sv4si"
2407   [
2408    (set (match_operand:DI 0 "s_register_operand" "=r")
2409         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2410                     (match_operand:V4SI 2 "s_register_operand" "w")]
2411          VRMLALDAVHXQ_S))
2412   ]
2413   "TARGET_HAVE_MVE"
2414   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
2415   [(set_attr "type" "mve_move")
2416 ])
2417
2418 ;;
2419 ;; [vrmlsldavhq_s])
2420 ;;
2421 (define_insn "mve_vrmlsldavhq_sv4si"
2422   [
2423    (set (match_operand:DI 0 "s_register_operand" "=r")
2424         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2425                     (match_operand:V4SI 2 "s_register_operand" "w")]
2426          VRMLSLDAVHQ_S))
2427   ]
2428   "TARGET_HAVE_MVE"
2429   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
2430   [(set_attr "type" "mve_move")
2431 ])
2432
2433 ;;
2434 ;; [vrmlsldavhxq_s])
2435 ;;
2436 (define_insn "mve_vrmlsldavhxq_sv4si"
2437   [
2438    (set (match_operand:DI 0 "s_register_operand" "=r")
2439         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2440                     (match_operand:V4SI 2 "s_register_operand" "w")]
2441          VRMLSLDAVHXQ_S))
2442   ]
2443   "TARGET_HAVE_MVE"
2444   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
2445   [(set_attr "type" "mve_move")
2446 ])
2447
2448 ;;
2449 ;; [vshllbq_n_s, vshllbq_n_u])
2450 ;;
2451 (define_insn "mve_vshllbq_n_<supf><mode>"
2452   [
2453    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2454         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2455                                   (match_operand:SI 2 "immediate_operand" "i")]
2456          VSHLLBQ_N))
2457   ]
2458   "TARGET_HAVE_MVE"
2459   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2460   [(set_attr "type" "mve_move")
2461 ])
2462
2463 ;;
2464 ;; [vshlltq_n_u, vshlltq_n_s])
2465 ;;
2466 (define_insn "mve_vshlltq_n_<supf><mode>"
2467   [
2468    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2469         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2470                                   (match_operand:SI 2 "immediate_operand" "i")]
2471          VSHLLTQ_N))
2472   ]
2473   "TARGET_HAVE_MVE"
2474   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2475   [(set_attr "type" "mve_move")
2476 ])
2477
2478 ;;
2479 ;; [vsubq_f])
2480 ;;
2481 (define_insn "mve_vsubq_f<mode>"
2482   [
2483    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2484         (minus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
2485                      (match_operand:MVE_0 2 "s_register_operand" "w")))
2486   ]
2487   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2488   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
2489   [(set_attr "type" "mve_move")
2490 ])
2491
2492 ;;
2493 ;; [vmulltq_poly_p])
2494 ;;
2495 (define_insn "mve_vmulltq_poly_p<mode>"
2496   [
2497    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2498         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2499                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2500          VMULLTQ_POLY_P))
2501   ]
2502   "TARGET_HAVE_MVE"
2503   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
2504   [(set_attr "type" "mve_move")
2505 ])
2506
2507 ;;
2508 ;; [vmullbq_poly_p])
2509 ;;
2510 (define_insn "mve_vmullbq_poly_p<mode>"
2511   [
2512    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2513         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
2514                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
2515          VMULLBQ_POLY_P))
2516   ]
2517   "TARGET_HAVE_MVE"
2518   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
2519   [(set_attr "type" "mve_move")
2520 ])
2521
2522 ;;
2523 ;; [vrmlaldavhq_u vrmlaldavhq_s])
2524 ;;
2525 (define_insn "mve_vrmlaldavhq_<supf>v4si"
2526   [
2527    (set (match_operand:DI 0 "s_register_operand" "=r")
2528         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
2529                     (match_operand:V4SI 2 "s_register_operand" "w")]
2530          VRMLALDAVHQ))
2531   ]
2532   "TARGET_HAVE_MVE"
2533   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
2534   [(set_attr "type" "mve_move")
2535 ])
2536
2537 ;;
2538 ;; [vbicq_m_n_s, vbicq_m_n_u])
2539 ;;
2540 (define_insn "mve_vbicq_m_n_<supf><mode>"
2541   [
2542    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2543         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2544                        (match_operand:SI 2 "immediate_operand" "i")
2545                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2546          VBICQ_M_N))
2547   ]
2548   "TARGET_HAVE_MVE"
2549   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
2550   [(set_attr "type" "mve_move")
2551    (set_attr "length""8")])
2552 ;;
2553 ;; [vcmpeqq_m_f])
2554 ;;
2555 (define_insn "mve_vcmpeqq_m_f<mode>"
2556   [
2557    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2558         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2559                     (match_operand:MVE_0 2 "s_register_operand" "w")
2560                     (match_operand:HI 3 "vpr_register_operand" "Up")]
2561          VCMPEQQ_M_F))
2562   ]
2563   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2564   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
2565   [(set_attr "type" "mve_move")
2566    (set_attr "length""8")])
2567 ;;
2568 ;; [vcvtaq_m_u, vcvtaq_m_s])
2569 ;;
2570 (define_insn "mve_vcvtaq_m_<supf><mode>"
2571   [
2572    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2573         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2574                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2575                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2576          VCVTAQ_M))
2577   ]
2578   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2579   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
2580   [(set_attr "type" "mve_move")
2581    (set_attr "length""8")])
2582 ;;
2583 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
2584 ;;
2585 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
2586   [
2587    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2588         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2589                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
2590                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2591          VCVTQ_M_TO_F))
2592   ]
2593   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2594   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
2595   [(set_attr "type" "mve_move")
2596    (set_attr "length""8")])
2597 ;;
2598 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
2599 ;;
2600 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
2601   [
2602    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2603         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2604                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2605                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2606          VQRSHRNBQ_N))
2607   ]
2608   "TARGET_HAVE_MVE"
2609   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
2610   [(set_attr "type" "mve_move")
2611 ])
2612 ;;
2613 ;; [vqrshrunbq_n_s])
2614 ;;
2615 (define_insn "mve_vqrshrunbq_n_s<mode>"
2616   [
2617    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
2618         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
2619                                  (match_operand:MVE_5 2 "s_register_operand" "w")
2620                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
2621          VQRSHRUNBQ_N_S))
2622   ]
2623   "TARGET_HAVE_MVE"
2624   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
2625   [(set_attr "type" "mve_move")
2626 ])
2627 ;;
2628 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
2629 ;;
2630 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
2631   [
2632    (set (match_operand:DI 0 "s_register_operand" "=r")
2633         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2634                     (match_operand:V4SI 2 "s_register_operand" "w")
2635                     (match_operand:V4SI 3 "s_register_operand" "w")]
2636          VRMLALDAVHAQ))
2637   ]
2638   "TARGET_HAVE_MVE"
2639   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
2640   [(set_attr "type" "mve_move")
2641 ])
2642
2643 ;;
2644 ;; [vabavq_s, vabavq_u])
2645 ;;
2646 (define_insn "mve_vabavq_<supf><mode>"
2647   [
2648    (set (match_operand:SI 0 "s_register_operand" "=r")
2649         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2650                     (match_operand:MVE_2 2 "s_register_operand" "w")
2651                     (match_operand:MVE_2 3 "s_register_operand" "w")]
2652          VABAVQ))
2653   ]
2654   "TARGET_HAVE_MVE"
2655   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
2656   [(set_attr "type" "mve_move")
2657 ])
2658
2659 ;;
2660 ;; [vshlcq_u vshlcq_s]
2661 ;;
2662 (define_expand "mve_vshlcq_vec_<supf><mode>"
2663  [(match_operand:MVE_2 0 "s_register_operand")
2664   (match_operand:MVE_2 1 "s_register_operand")
2665   (match_operand:SI 2 "s_register_operand")
2666   (match_operand:SI 3 "mve_imm_32")
2667   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2668  "TARGET_HAVE_MVE"
2669 {
2670   rtx ignore_wb = gen_reg_rtx (SImode);
2671   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
2672                                       operands[2], operands[3]));
2673   DONE;
2674 })
2675
2676 (define_expand "mve_vshlcq_carry_<supf><mode>"
2677  [(match_operand:SI 0 "s_register_operand")
2678   (match_operand:MVE_2 1 "s_register_operand")
2679   (match_operand:SI 2 "s_register_operand")
2680   (match_operand:SI 3 "mve_imm_32")
2681   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
2682  "TARGET_HAVE_MVE"
2683 {
2684   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
2685   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
2686                                       operands[2], operands[3]));
2687   DONE;
2688 })
2689
2690 (define_insn "mve_vshlcq_<supf><mode>"
2691  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
2692        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
2693                       (match_operand:SI 3 "s_register_operand" "1")
2694                       (match_operand:SI 4 "mve_imm_32" "Rf")]
2695         VSHLCQ))
2696   (set (match_operand:SI  1 "s_register_operand" "=r")
2697        (unspec:SI [(match_dup 2)
2698                    (match_dup 3)
2699                    (match_dup 4)]
2700         VSHLCQ))]
2701  "TARGET_HAVE_MVE"
2702  "vshlc %q0, %1, %4")
2703
2704 ;;
2705 ;; [vabsq_m_s])
2706 ;;
2707 (define_insn "mve_vabsq_m_s<mode>"
2708   [
2709    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2710         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2711                        (match_operand:MVE_2 2 "s_register_operand" "w")
2712                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2713          VABSQ_M_S))
2714   ]
2715   "TARGET_HAVE_MVE"
2716   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
2717   [(set_attr "type" "mve_move")
2718    (set_attr "length""8")])
2719
2720 ;;
2721 ;; [vaddvaq_p_u, vaddvaq_p_s])
2722 ;;
2723 (define_insn "mve_vaddvaq_p_<supf><mode>"
2724   [
2725    (set (match_operand:SI 0 "s_register_operand" "=Te")
2726         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
2727                        (match_operand:MVE_2 2 "s_register_operand" "w")
2728                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2729          VADDVAQ_P))
2730   ]
2731   "TARGET_HAVE_MVE"
2732   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
2733   [(set_attr "type" "mve_move")
2734    (set_attr "length""8")])
2735
2736 ;;
2737 ;; [vclsq_m_s])
2738 ;;
2739 (define_insn "mve_vclsq_m_s<mode>"
2740   [
2741    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2742         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2743                        (match_operand:MVE_2 2 "s_register_operand" "w")
2744                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2745          VCLSQ_M_S))
2746   ]
2747   "TARGET_HAVE_MVE"
2748   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
2749   [(set_attr "type" "mve_move")
2750    (set_attr "length""8")])
2751
2752 ;;
2753 ;; [vclzq_m_s, vclzq_m_u])
2754 ;;
2755 (define_insn "mve_vclzq_m_<supf><mode>"
2756   [
2757    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2758         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2759                        (match_operand:MVE_2 2 "s_register_operand" "w")
2760                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2761          VCLZQ_M))
2762   ]
2763   "TARGET_HAVE_MVE"
2764   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
2765   [(set_attr "type" "mve_move")
2766    (set_attr "length""8")])
2767
2768 ;;
2769 ;; [vcmpcsq_m_n_u])
2770 ;;
2771 (define_insn "mve_vcmpcsq_m_n_u<mode>"
2772   [
2773    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2774         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2775                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2776                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2777          VCMPCSQ_M_N_U))
2778   ]
2779   "TARGET_HAVE_MVE"
2780   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
2781   [(set_attr "type" "mve_move")
2782    (set_attr "length""8")])
2783
2784 ;;
2785 ;; [vcmpcsq_m_u])
2786 ;;
2787 (define_insn "mve_vcmpcsq_m_u<mode>"
2788   [
2789    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2790         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2791                        (match_operand:MVE_2 2 "s_register_operand" "w")
2792                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2793          VCMPCSQ_M_U))
2794   ]
2795   "TARGET_HAVE_MVE"
2796   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
2797   [(set_attr "type" "mve_move")
2798    (set_attr "length""8")])
2799
2800 ;;
2801 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
2802 ;;
2803 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
2804   [
2805    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2806         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2807                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2808                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2809          VCMPEQQ_M_N))
2810   ]
2811   "TARGET_HAVE_MVE"
2812   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
2813   [(set_attr "type" "mve_move")
2814    (set_attr "length""8")])
2815
2816 ;;
2817 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
2818 ;;
2819 (define_insn "mve_vcmpeqq_m_<supf><mode>"
2820   [
2821    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2822         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2823                        (match_operand:MVE_2 2 "s_register_operand" "w")
2824                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2825          VCMPEQQ_M))
2826   ]
2827   "TARGET_HAVE_MVE"
2828   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
2829   [(set_attr "type" "mve_move")
2830    (set_attr "length""8")])
2831
2832 ;;
2833 ;; [vcmpgeq_m_n_s])
2834 ;;
2835 (define_insn "mve_vcmpgeq_m_n_s<mode>"
2836   [
2837    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2838         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2839                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2840                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2841          VCMPGEQ_M_N_S))
2842   ]
2843   "TARGET_HAVE_MVE"
2844   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
2845   [(set_attr "type" "mve_move")
2846    (set_attr "length""8")])
2847
2848 ;;
2849 ;; [vcmpgeq_m_s])
2850 ;;
2851 (define_insn "mve_vcmpgeq_m_s<mode>"
2852   [
2853    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2854         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2855                        (match_operand:MVE_2 2 "s_register_operand" "w")
2856                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2857          VCMPGEQ_M_S))
2858   ]
2859   "TARGET_HAVE_MVE"
2860   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
2861   [(set_attr "type" "mve_move")
2862    (set_attr "length""8")])
2863
2864 ;;
2865 ;; [vcmpgtq_m_n_s])
2866 ;;
2867 (define_insn "mve_vcmpgtq_m_n_s<mode>"
2868   [
2869    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2870         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2871                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2872                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2873          VCMPGTQ_M_N_S))
2874   ]
2875   "TARGET_HAVE_MVE"
2876   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
2877   [(set_attr "type" "mve_move")
2878    (set_attr "length""8")])
2879
2880 ;;
2881 ;; [vcmpgtq_m_s])
2882 ;;
2883 (define_insn "mve_vcmpgtq_m_s<mode>"
2884   [
2885    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2886         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2887                        (match_operand:MVE_2 2 "s_register_operand" "w")
2888                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2889          VCMPGTQ_M_S))
2890   ]
2891   "TARGET_HAVE_MVE"
2892   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
2893   [(set_attr "type" "mve_move")
2894    (set_attr "length""8")])
2895
2896 ;;
2897 ;; [vcmphiq_m_n_u])
2898 ;;
2899 (define_insn "mve_vcmphiq_m_n_u<mode>"
2900   [
2901    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2902         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2903                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2904                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2905          VCMPHIQ_M_N_U))
2906   ]
2907   "TARGET_HAVE_MVE"
2908   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
2909   [(set_attr "type" "mve_move")
2910    (set_attr "length""8")])
2911
2912 ;;
2913 ;; [vcmphiq_m_u])
2914 ;;
2915 (define_insn "mve_vcmphiq_m_u<mode>"
2916   [
2917    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2918         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2919                        (match_operand:MVE_2 2 "s_register_operand" "w")
2920                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2921          VCMPHIQ_M_U))
2922   ]
2923   "TARGET_HAVE_MVE"
2924   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
2925   [(set_attr "type" "mve_move")
2926    (set_attr "length""8")])
2927
2928 ;;
2929 ;; [vcmpleq_m_n_s])
2930 ;;
2931 (define_insn "mve_vcmpleq_m_n_s<mode>"
2932   [
2933    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2934         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2935                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2936                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2937          VCMPLEQ_M_N_S))
2938   ]
2939   "TARGET_HAVE_MVE"
2940   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
2941   [(set_attr "type" "mve_move")
2942    (set_attr "length""8")])
2943
2944 ;;
2945 ;; [vcmpleq_m_s])
2946 ;;
2947 (define_insn "mve_vcmpleq_m_s<mode>"
2948   [
2949    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2950         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2951                        (match_operand:MVE_2 2 "s_register_operand" "w")
2952                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2953          VCMPLEQ_M_S))
2954   ]
2955   "TARGET_HAVE_MVE"
2956   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
2957   [(set_attr "type" "mve_move")
2958    (set_attr "length""8")])
2959
2960 ;;
2961 ;; [vcmpltq_m_n_s])
2962 ;;
2963 (define_insn "mve_vcmpltq_m_n_s<mode>"
2964   [
2965    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2966         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2967                        (match_operand:<V_elem> 2 "s_register_operand" "r")
2968                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2969          VCMPLTQ_M_N_S))
2970   ]
2971   "TARGET_HAVE_MVE"
2972   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
2973   [(set_attr "type" "mve_move")
2974    (set_attr "length""8")])
2975
2976 ;;
2977 ;; [vcmpltq_m_s])
2978 ;;
2979 (define_insn "mve_vcmpltq_m_s<mode>"
2980   [
2981    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2982         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2983                        (match_operand:MVE_2 2 "s_register_operand" "w")
2984                        (match_operand:HI 3 "vpr_register_operand" "Up")]
2985          VCMPLTQ_M_S))
2986   ]
2987   "TARGET_HAVE_MVE"
2988   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
2989   [(set_attr "type" "mve_move")
2990    (set_attr "length""8")])
2991
2992 ;;
2993 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
2994 ;;
2995 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
2996   [
2997    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2998         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
2999                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3000                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3001          VCMPNEQ_M_N))
3002   ]
3003   "TARGET_HAVE_MVE"
3004   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3005   [(set_attr "type" "mve_move")
3006    (set_attr "length""8")])
3007
3008 ;;
3009 ;; [vcmpneq_m_s, vcmpneq_m_u])
3010 ;;
3011 (define_insn "mve_vcmpneq_m_<supf><mode>"
3012   [
3013    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3014         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3015                        (match_operand:MVE_2 2 "s_register_operand" "w")
3016                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3017          VCMPNEQ_M))
3018   ]
3019   "TARGET_HAVE_MVE"
3020   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3021   [(set_attr "type" "mve_move")
3022    (set_attr "length""8")])
3023
3024 ;;
3025 ;; [vdupq_m_n_s, vdupq_m_n_u])
3026 ;;
3027 (define_insn "mve_vdupq_m_n_<supf><mode>"
3028   [
3029    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3030         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3031                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3032                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3033          VDUPQ_M_N))
3034   ]
3035   "TARGET_HAVE_MVE"
3036   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3037   [(set_attr "type" "mve_move")
3038    (set_attr "length""8")])
3039
3040 ;;
3041 ;; [vmaxaq_m_s])
3042 ;;
3043 (define_insn "mve_vmaxaq_m_s<mode>"
3044   [
3045    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3046         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3047                        (match_operand:MVE_2 2 "s_register_operand" "w")
3048                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3049          VMAXAQ_M_S))
3050   ]
3051   "TARGET_HAVE_MVE"
3052   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
3053   [(set_attr "type" "mve_move")
3054    (set_attr "length""8")])
3055
3056 ;;
3057 ;; [vmaxavq_p_s])
3058 ;;
3059 (define_insn "mve_vmaxavq_p_s<mode>"
3060   [
3061    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3062         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3063                        (match_operand:MVE_2 2 "s_register_operand" "w")
3064                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3065          VMAXAVQ_P_S))
3066   ]
3067   "TARGET_HAVE_MVE"
3068   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
3069   [(set_attr "type" "mve_move")
3070    (set_attr "length""8")])
3071
3072 ;;
3073 ;; [vmaxvq_p_u, vmaxvq_p_s])
3074 ;;
3075 (define_insn "mve_vmaxvq_p_<supf><mode>"
3076   [
3077    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3078         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3079                        (match_operand:MVE_2 2 "s_register_operand" "w")
3080                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3081          VMAXVQ_P))
3082   ]
3083   "TARGET_HAVE_MVE"
3084   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
3085   [(set_attr "type" "mve_move")
3086    (set_attr "length""8")])
3087
3088 ;;
3089 ;; [vminaq_m_s])
3090 ;;
3091 (define_insn "mve_vminaq_m_s<mode>"
3092   [
3093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3095                        (match_operand:MVE_2 2 "s_register_operand" "w")
3096                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3097          VMINAQ_M_S))
3098   ]
3099   "TARGET_HAVE_MVE"
3100   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
3101   [(set_attr "type" "mve_move")
3102    (set_attr "length""8")])
3103
3104 ;;
3105 ;; [vminavq_p_s])
3106 ;;
3107 (define_insn "mve_vminavq_p_s<mode>"
3108   [
3109    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3110         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3111                        (match_operand:MVE_2 2 "s_register_operand" "w")
3112                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3113          VMINAVQ_P_S))
3114   ]
3115   "TARGET_HAVE_MVE"
3116   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
3117   [(set_attr "type" "mve_move")
3118    (set_attr "length""8")])
3119
3120 ;;
3121 ;; [vminvq_p_s, vminvq_p_u])
3122 ;;
3123 (define_insn "mve_vminvq_p_<supf><mode>"
3124   [
3125    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3126         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3127                        (match_operand:MVE_2 2 "s_register_operand" "w")
3128                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3129          VMINVQ_P))
3130   ]
3131   "TARGET_HAVE_MVE"
3132   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
3133   [(set_attr "type" "mve_move")
3134    (set_attr "length""8")])
3135
3136 ;;
3137 ;; [vmladavaq_u, vmladavaq_s])
3138 ;;
3139 (define_insn "mve_vmladavaq_<supf><mode>"
3140   [
3141    (set (match_operand:SI 0 "s_register_operand" "=Te")
3142         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3143                        (match_operand:MVE_2 2 "s_register_operand" "w")
3144                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3145          VMLADAVAQ))
3146   ]
3147   "TARGET_HAVE_MVE"
3148   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
3149   [(set_attr "type" "mve_move")
3150 ])
3151
3152 ;;
3153 ;; [vmladavq_p_u, vmladavq_p_s])
3154 ;;
3155 (define_insn "mve_vmladavq_p_<supf><mode>"
3156   [
3157    (set (match_operand:SI 0 "s_register_operand" "=Te")
3158         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3159                        (match_operand:MVE_2 2 "s_register_operand" "w")
3160                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3161          VMLADAVQ_P))
3162   ]
3163   "TARGET_HAVE_MVE"
3164   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
3165   [(set_attr "type" "mve_move")
3166    (set_attr "length""8")])
3167
3168 ;;
3169 ;; [vmladavxq_p_s])
3170 ;;
3171 (define_insn "mve_vmladavxq_p_s<mode>"
3172   [
3173    (set (match_operand:SI 0 "s_register_operand" "=Te")
3174         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3175                        (match_operand:MVE_2 2 "s_register_operand" "w")
3176                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3177          VMLADAVXQ_P_S))
3178   ]
3179   "TARGET_HAVE_MVE"
3180   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
3181   [(set_attr "type" "mve_move")
3182    (set_attr "length""8")])
3183
3184 ;;
3185 ;; [vmlaq_n_u, vmlaq_n_s])
3186 ;;
3187 (define_insn "mve_vmlaq_n_<supf><mode>"
3188   [
3189    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3190         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3191                        (match_operand:MVE_2 2 "s_register_operand" "w")
3192                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3193          VMLAQ_N))
3194   ]
3195   "TARGET_HAVE_MVE"
3196   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
3197   [(set_attr "type" "mve_move")
3198 ])
3199
3200 ;;
3201 ;; [vmlasq_n_u, vmlasq_n_s])
3202 ;;
3203 (define_insn "mve_vmlasq_n_<supf><mode>"
3204   [
3205    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3206         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3207                        (match_operand:MVE_2 2 "s_register_operand" "w")
3208                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3209          VMLASQ_N))
3210   ]
3211   "TARGET_HAVE_MVE"
3212   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
3213   [(set_attr "type" "mve_move")
3214 ])
3215
3216 ;;
3217 ;; [vmlsdavq_p_s])
3218 ;;
3219 (define_insn "mve_vmlsdavq_p_s<mode>"
3220   [
3221    (set (match_operand:SI 0 "s_register_operand" "=Te")
3222         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3223                        (match_operand:MVE_2 2 "s_register_operand" "w")
3224                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3225          VMLSDAVQ_P_S))
3226   ]
3227   "TARGET_HAVE_MVE"
3228   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
3229   [(set_attr "type" "mve_move")
3230    (set_attr "length""8")])
3231
3232 ;;
3233 ;; [vmlsdavxq_p_s])
3234 ;;
3235 (define_insn "mve_vmlsdavxq_p_s<mode>"
3236   [
3237    (set (match_operand:SI 0 "s_register_operand" "=Te")
3238         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
3239                        (match_operand:MVE_2 2 "s_register_operand" "w")
3240                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3241          VMLSDAVXQ_P_S))
3242   ]
3243   "TARGET_HAVE_MVE"
3244   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
3245   [(set_attr "type" "mve_move")
3246    (set_attr "length""8")])
3247
3248 ;;
3249 ;; [vmvnq_m_s, vmvnq_m_u])
3250 ;;
3251 (define_insn "mve_vmvnq_m_<supf><mode>"
3252   [
3253    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3254         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3255                        (match_operand:MVE_2 2 "s_register_operand" "w")
3256                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3257          VMVNQ_M))
3258   ]
3259   "TARGET_HAVE_MVE"
3260   "vpst\;vmvnt %q0, %q2"
3261   [(set_attr "type" "mve_move")
3262    (set_attr "length""8")])
3263
3264 ;;
3265 ;; [vnegq_m_s])
3266 ;;
3267 (define_insn "mve_vnegq_m_s<mode>"
3268   [
3269    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3270         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3271                        (match_operand:MVE_2 2 "s_register_operand" "w")
3272                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3273          VNEGQ_M_S))
3274   ]
3275   "TARGET_HAVE_MVE"
3276   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
3277   [(set_attr "type" "mve_move")
3278    (set_attr "length""8")])
3279
3280 ;;
3281 ;; [vpselq_u, vpselq_s])
3282 ;;
3283 (define_insn "@mve_vpselq_<supf><mode>"
3284   [
3285    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
3286         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
3287                        (match_operand:MVE_1 2 "s_register_operand" "w")
3288                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3289          VPSELQ))
3290   ]
3291   "TARGET_HAVE_MVE"
3292   "vpsel %q0, %q1, %q2"
3293   [(set_attr "type" "mve_move")
3294 ])
3295
3296 ;;
3297 ;; [vqabsq_m_s])
3298 ;;
3299 (define_insn "mve_vqabsq_m_s<mode>"
3300   [
3301    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3302         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3303                        (match_operand:MVE_2 2 "s_register_operand" "w")
3304                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3305          VQABSQ_M_S))
3306   ]
3307   "TARGET_HAVE_MVE"
3308   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
3309   [(set_attr "type" "mve_move")
3310    (set_attr "length""8")])
3311
3312 ;;
3313 ;; [vqdmlahq_n_s])
3314 ;;
3315 (define_insn "mve_vqdmlahq_n_<supf><mode>"
3316   [
3317    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3318         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3319                        (match_operand:MVE_2 2 "s_register_operand" "w")
3320                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3321          VQDMLAHQ_N))
3322   ]
3323   "TARGET_HAVE_MVE"
3324   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3325   [(set_attr "type" "mve_move")
3326 ])
3327
3328 ;;
3329 ;; [vqdmlashq_n_s])
3330 ;;
3331 (define_insn "mve_vqdmlashq_n_<supf><mode>"
3332   [
3333    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3334         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3335                        (match_operand:MVE_2 2 "s_register_operand" "w")
3336                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3337          VQDMLASHQ_N))
3338   ]
3339   "TARGET_HAVE_MVE"
3340   "vqdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3341   [(set_attr "type" "mve_move")
3342 ])
3343
3344 ;;
3345 ;; [vqnegq_m_s])
3346 ;;
3347 (define_insn "mve_vqnegq_m_s<mode>"
3348   [
3349    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3350         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3351                        (match_operand:MVE_2 2 "s_register_operand" "w")
3352                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3353          VQNEGQ_M_S))
3354   ]
3355   "TARGET_HAVE_MVE"
3356   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
3357   [(set_attr "type" "mve_move")
3358    (set_attr "length""8")])
3359
3360 ;;
3361 ;; [vqrdmladhq_s])
3362 ;;
3363 (define_insn "mve_vqrdmladhq_s<mode>"
3364   [
3365    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3366         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3367                        (match_operand:MVE_2 2 "s_register_operand" "w")
3368                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3369          VQRDMLADHQ_S))
3370   ]
3371   "TARGET_HAVE_MVE"
3372   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3373   [(set_attr "type" "mve_move")
3374 ])
3375
3376 ;;
3377 ;; [vqrdmladhxq_s])
3378 ;;
3379 (define_insn "mve_vqrdmladhxq_s<mode>"
3380   [
3381    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3382         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3383                        (match_operand:MVE_2 2 "s_register_operand" "w")
3384                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3385          VQRDMLADHXQ_S))
3386   ]
3387   "TARGET_HAVE_MVE"
3388   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3389   [(set_attr "type" "mve_move")
3390 ])
3391
3392 ;;
3393 ;; [vqrdmlahq_n_s])
3394 ;;
3395 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
3396   [
3397    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3398         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3399                        (match_operand:MVE_2 2 "s_register_operand" "w")
3400                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3401          VQRDMLAHQ_N))
3402   ]
3403   "TARGET_HAVE_MVE"
3404   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
3405   [(set_attr "type" "mve_move")
3406 ])
3407
3408 ;;
3409 ;; [vqrdmlashq_n_s])
3410 ;;
3411 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
3412   [
3413    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3414         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3415                        (match_operand:MVE_2 2 "s_register_operand" "w")
3416                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
3417          VQRDMLASHQ_N))
3418   ]
3419   "TARGET_HAVE_MVE"
3420   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
3421   [(set_attr "type" "mve_move")
3422 ])
3423
3424 ;;
3425 ;; [vqrdmlsdhq_s])
3426 ;;
3427 (define_insn "mve_vqrdmlsdhq_s<mode>"
3428   [
3429    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3430         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3431                        (match_operand:MVE_2 2 "s_register_operand" "w")
3432                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3433          VQRDMLSDHQ_S))
3434   ]
3435   "TARGET_HAVE_MVE"
3436   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3437   [(set_attr "type" "mve_move")
3438 ])
3439
3440 ;;
3441 ;; [vqrdmlsdhxq_s])
3442 ;;
3443 (define_insn "mve_vqrdmlsdhxq_s<mode>"
3444   [
3445    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3446         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3447                        (match_operand:MVE_2 2 "s_register_operand" "w")
3448                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3449          VQRDMLSDHXQ_S))
3450   ]
3451   "TARGET_HAVE_MVE"
3452   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3453   [(set_attr "type" "mve_move")
3454 ])
3455
3456 ;;
3457 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
3458 ;;
3459 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
3460   [
3461    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3462         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3463                        (match_operand:SI 2 "s_register_operand" "r")
3464                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3465          VQRSHLQ_M_N))
3466   ]
3467   "TARGET_HAVE_MVE"
3468   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
3469   [(set_attr "type" "mve_move")
3470    (set_attr "length""8")])
3471
3472 ;;
3473 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
3474 ;;
3475 (define_insn "mve_vqshlq_m_r_<supf><mode>"
3476   [
3477    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3478         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3479                        (match_operand:SI 2 "s_register_operand" "r")
3480                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3481          VQSHLQ_M_R))
3482   ]
3483   "TARGET_HAVE_MVE"
3484   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3485   [(set_attr "type" "mve_move")
3486    (set_attr "length""8")])
3487
3488 ;;
3489 ;; [vrev64q_m_u, vrev64q_m_s])
3490 ;;
3491 (define_insn "mve_vrev64q_m_<supf><mode>"
3492   [
3493    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3494         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3495                        (match_operand:MVE_2 2 "s_register_operand" "w")
3496                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3497          VREV64Q_M))
3498   ]
3499   "TARGET_HAVE_MVE"
3500   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
3501   [(set_attr "type" "mve_move")
3502    (set_attr "length""8")])
3503
3504 ;;
3505 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
3506 ;;
3507 (define_insn "mve_vrshlq_m_n_<supf><mode>"
3508   [
3509    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3510         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3511                        (match_operand:SI 2 "s_register_operand" "r")
3512                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3513          VRSHLQ_M_N))
3514   ]
3515   "TARGET_HAVE_MVE"
3516   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3517   [(set_attr "type" "mve_move")
3518    (set_attr "length""8")])
3519
3520 ;;
3521 ;; [vshlq_m_r_u, vshlq_m_r_s])
3522 ;;
3523 (define_insn "mve_vshlq_m_r_<supf><mode>"
3524   [
3525    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3526         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3527                        (match_operand:SI 2 "s_register_operand" "r")
3528                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3529          VSHLQ_M_R))
3530   ]
3531   "TARGET_HAVE_MVE"
3532   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
3533   [(set_attr "type" "mve_move")
3534    (set_attr "length""8")])
3535
3536 ;;
3537 ;; [vsliq_n_u, vsliq_n_s])
3538 ;;
3539 (define_insn "mve_vsliq_n_<supf><mode>"
3540   [
3541    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3542         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3543                        (match_operand:MVE_2 2 "s_register_operand" "w")
3544                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
3545          VSLIQ_N))
3546   ]
3547   "TARGET_HAVE_MVE"
3548   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
3549   [(set_attr "type" "mve_move")
3550 ])
3551
3552 ;;
3553 ;; [vsriq_n_u, vsriq_n_s])
3554 ;;
3555 (define_insn "mve_vsriq_n_<supf><mode>"
3556   [
3557    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3558         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3559                        (match_operand:MVE_2 2 "s_register_operand" "w")
3560                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
3561          VSRIQ_N))
3562   ]
3563   "TARGET_HAVE_MVE"
3564   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
3565   [(set_attr "type" "mve_move")
3566 ])
3567
3568 ;;
3569 ;; [vqdmlsdhxq_s])
3570 ;;
3571 (define_insn "mve_vqdmlsdhxq_s<mode>"
3572   [
3573    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3574         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3575                        (match_operand:MVE_2 2 "s_register_operand" "w")
3576                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3577          VQDMLSDHXQ_S))
3578   ]
3579   "TARGET_HAVE_MVE"
3580   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3581   [(set_attr "type" "mve_move")
3582 ])
3583
3584 ;;
3585 ;; [vqdmlsdhq_s])
3586 ;;
3587 (define_insn "mve_vqdmlsdhq_s<mode>"
3588   [
3589    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3590         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3591                        (match_operand:MVE_2 2 "s_register_operand" "w")
3592                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3593          VQDMLSDHQ_S))
3594   ]
3595   "TARGET_HAVE_MVE"
3596   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3597   [(set_attr "type" "mve_move")
3598 ])
3599
3600 ;;
3601 ;; [vqdmladhxq_s])
3602 ;;
3603 (define_insn "mve_vqdmladhxq_s<mode>"
3604   [
3605    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3606         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3607                        (match_operand:MVE_2 2 "s_register_operand" "w")
3608                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3609          VQDMLADHXQ_S))
3610   ]
3611   "TARGET_HAVE_MVE"
3612   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
3613   [(set_attr "type" "mve_move")
3614 ])
3615
3616 ;;
3617 ;; [vqdmladhq_s])
3618 ;;
3619 (define_insn "mve_vqdmladhq_s<mode>"
3620   [
3621    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3622         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3623                        (match_operand:MVE_2 2 "s_register_operand" "w")
3624                        (match_operand:MVE_2 3 "s_register_operand" "w")]
3625          VQDMLADHQ_S))
3626   ]
3627   "TARGET_HAVE_MVE"
3628   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
3629   [(set_attr "type" "mve_move")
3630 ])
3631
3632 ;;
3633 ;; [vmlsdavaxq_s])
3634 ;;
3635 (define_insn "mve_vmlsdavaxq_s<mode>"
3636   [
3637    (set (match_operand:SI 0 "s_register_operand" "=Te")
3638         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3639                     (match_operand:MVE_2 2 "s_register_operand" "w")
3640                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3641          VMLSDAVAXQ_S))
3642   ]
3643   "TARGET_HAVE_MVE"
3644   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3645   [(set_attr "type" "mve_move")
3646 ])
3647
3648 ;;
3649 ;; [vmlsdavaq_s])
3650 ;;
3651 (define_insn "mve_vmlsdavaq_s<mode>"
3652   [
3653    (set (match_operand:SI 0 "s_register_operand" "=Te")
3654         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3655                     (match_operand:MVE_2 2 "s_register_operand" "w")
3656                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3657          VMLSDAVAQ_S))
3658   ]
3659   "TARGET_HAVE_MVE"
3660   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
3661   [(set_attr "type" "mve_move")
3662 ])
3663
3664 ;;
3665 ;; [vmladavaxq_s])
3666 ;;
3667 (define_insn "mve_vmladavaxq_s<mode>"
3668   [
3669    (set (match_operand:SI 0 "s_register_operand" "=Te")
3670         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3671                     (match_operand:MVE_2 2 "s_register_operand" "w")
3672                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3673          VMLADAVAXQ_S))
3674   ]
3675   "TARGET_HAVE_MVE"
3676   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
3677   [(set_attr "type" "mve_move")
3678 ])
3679 ;;
3680 ;; [vabsq_m_f])
3681 ;;
3682 (define_insn "mve_vabsq_m_f<mode>"
3683   [
3684    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3685         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3686                        (match_operand:MVE_0 2 "s_register_operand" "w")
3687                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3688          VABSQ_M_F))
3689   ]
3690   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3691   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
3692   [(set_attr "type" "mve_move")
3693    (set_attr "length""8")])
3694
3695 ;;
3696 ;; [vaddlvaq_p_s vaddlvaq_p_u])
3697 ;;
3698 (define_insn "mve_vaddlvaq_p_<supf>v4si"
3699   [
3700    (set (match_operand:DI 0 "s_register_operand" "=r")
3701         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3702                        (match_operand:V4SI 2 "s_register_operand" "w")
3703                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3704          VADDLVAQ_P))
3705   ]
3706   "TARGET_HAVE_MVE"
3707   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
3708   [(set_attr "type" "mve_move")
3709    (set_attr "length""8")])
3710 ;;
3711 ;; [vcmlaq, vcmlaq_rot90, vcmlaq_rot180, vcmlaq_rot270])
3712 ;;
3713 (define_insn "mve_vcmlaq<mve_rot><mode>"
3714   [
3715    (set (match_operand:MVE_0 0 "s_register_operand" "=w,w")
3716         (plus:MVE_0 (match_operand:MVE_0 1 "reg_or_zero_operand" "Dz,0")
3717                     (unspec:MVE_0
3718                         [(match_operand:MVE_0 2 "s_register_operand" "w,w")
3719                          (match_operand:MVE_0 3 "s_register_operand" "w,w")]
3720                      VCMLA)))
3721   ]
3722   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3723   "@
3724    vcmul.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>
3725    vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #<rot>"
3726   [(set_attr "type" "mve_move")
3727 ])
3728
3729 ;;
3730 ;; [vcmpeqq_m_n_f])
3731 ;;
3732 (define_insn "mve_vcmpeqq_m_n_f<mode>"
3733   [
3734    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3735         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3736                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3737                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3738          VCMPEQQ_M_N_F))
3739   ]
3740   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3741   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
3742   [(set_attr "type" "mve_move")
3743    (set_attr "length""8")])
3744
3745 ;;
3746 ;; [vcmpgeq_m_f])
3747 ;;
3748 (define_insn "mve_vcmpgeq_m_f<mode>"
3749   [
3750    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3751         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3752                        (match_operand:MVE_0 2 "s_register_operand" "w")
3753                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3754          VCMPGEQ_M_F))
3755   ]
3756   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3757   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
3758   [(set_attr "type" "mve_move")
3759    (set_attr "length""8")])
3760
3761 ;;
3762 ;; [vcmpgeq_m_n_f])
3763 ;;
3764 (define_insn "mve_vcmpgeq_m_n_f<mode>"
3765   [
3766    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3767         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3768                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3769                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3770          VCMPGEQ_M_N_F))
3771   ]
3772   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3773   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
3774   [(set_attr "type" "mve_move")
3775    (set_attr "length""8")])
3776
3777 ;;
3778 ;; [vcmpgtq_m_f])
3779 ;;
3780 (define_insn "mve_vcmpgtq_m_f<mode>"
3781   [
3782    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3783         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3784                        (match_operand:MVE_0 2 "s_register_operand" "w")
3785                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3786          VCMPGTQ_M_F))
3787   ]
3788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3789   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
3790   [(set_attr "type" "mve_move")
3791    (set_attr "length""8")])
3792
3793 ;;
3794 ;; [vcmpgtq_m_n_f])
3795 ;;
3796 (define_insn "mve_vcmpgtq_m_n_f<mode>"
3797   [
3798    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3799         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3800                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3801                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3802          VCMPGTQ_M_N_F))
3803   ]
3804   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3805   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
3806   [(set_attr "type" "mve_move")
3807    (set_attr "length""8")])
3808
3809 ;;
3810 ;; [vcmpleq_m_f])
3811 ;;
3812 (define_insn "mve_vcmpleq_m_f<mode>"
3813   [
3814    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3815         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3816                        (match_operand:MVE_0 2 "s_register_operand" "w")
3817                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3818          VCMPLEQ_M_F))
3819   ]
3820   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3821   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
3822   [(set_attr "type" "mve_move")
3823    (set_attr "length""8")])
3824
3825 ;;
3826 ;; [vcmpleq_m_n_f])
3827 ;;
3828 (define_insn "mve_vcmpleq_m_n_f<mode>"
3829   [
3830    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3831         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3832                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3833                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3834          VCMPLEQ_M_N_F))
3835   ]
3836   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3837   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
3838   [(set_attr "type" "mve_move")
3839    (set_attr "length""8")])
3840
3841 ;;
3842 ;; [vcmpltq_m_f])
3843 ;;
3844 (define_insn "mve_vcmpltq_m_f<mode>"
3845   [
3846    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3847         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3848                        (match_operand:MVE_0 2 "s_register_operand" "w")
3849                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3850          VCMPLTQ_M_F))
3851   ]
3852   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3853   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
3854   [(set_attr "type" "mve_move")
3855    (set_attr "length""8")])
3856
3857 ;;
3858 ;; [vcmpltq_m_n_f])
3859 ;;
3860 (define_insn "mve_vcmpltq_m_n_f<mode>"
3861   [
3862    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3863         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3864                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3865                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3866          VCMPLTQ_M_N_F))
3867   ]
3868   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3869   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
3870   [(set_attr "type" "mve_move")
3871    (set_attr "length""8")])
3872
3873 ;;
3874 ;; [vcmpneq_m_f])
3875 ;;
3876 (define_insn "mve_vcmpneq_m_f<mode>"
3877   [
3878    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3879         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3880                        (match_operand:MVE_0 2 "s_register_operand" "w")
3881                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3882          VCMPNEQ_M_F))
3883   ]
3884   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3885   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
3886   [(set_attr "type" "mve_move")
3887    (set_attr "length""8")])
3888
3889 ;;
3890 ;; [vcmpneq_m_n_f])
3891 ;;
3892 (define_insn "mve_vcmpneq_m_n_f<mode>"
3893   [
3894    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3895         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3896                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3897                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3898          VCMPNEQ_M_N_F))
3899   ]
3900   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3901   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
3902   [(set_attr "type" "mve_move")
3903    (set_attr "length""8")])
3904
3905 ;;
3906 ;; [vcvtbq_m_f16_f32])
3907 ;;
3908 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
3909   [
3910    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3911         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3912                        (match_operand:V4SF 2 "s_register_operand" "w")
3913                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3914          VCVTBQ_M_F16_F32))
3915   ]
3916   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3917   "vpst\;vcvtbt.f16.f32 %q0, %q2"
3918   [(set_attr "type" "mve_move")
3919    (set_attr "length""8")])
3920
3921 ;;
3922 ;; [vcvtbq_m_f32_f16])
3923 ;;
3924 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
3925   [
3926    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3927         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3928                        (match_operand:V8HF 2 "s_register_operand" "w")
3929                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3930          VCVTBQ_M_F32_F16))
3931   ]
3932   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3933   "vpst\;vcvtbt.f32.f16 %q0, %q2"
3934   [(set_attr "type" "mve_move")
3935    (set_attr "length""8")])
3936
3937 ;;
3938 ;; [vcvttq_m_f16_f32])
3939 ;;
3940 (define_insn "mve_vcvttq_m_f16_f32v8hf"
3941   [
3942    (set (match_operand:V8HF 0 "s_register_operand" "=w")
3943         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
3944                        (match_operand:V4SF 2 "s_register_operand" "w")
3945                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3946          VCVTTQ_M_F16_F32))
3947   ]
3948   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3949   "vpst\;vcvttt.f16.f32 %q0, %q2"
3950   [(set_attr "type" "mve_move")
3951    (set_attr "length""8")])
3952
3953 ;;
3954 ;; [vcvttq_m_f32_f16])
3955 ;;
3956 (define_insn "mve_vcvttq_m_f32_f16v4sf"
3957   [
3958    (set (match_operand:V4SF 0 "s_register_operand" "=w")
3959         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
3960                        (match_operand:V8HF 2 "s_register_operand" "w")
3961                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3962          VCVTTQ_M_F32_F16))
3963   ]
3964   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3965   "vpst\;vcvttt.f32.f16 %q0, %q2"
3966   [(set_attr "type" "mve_move")
3967    (set_attr "length""8")])
3968
3969 ;;
3970 ;; [vdupq_m_n_f])
3971 ;;
3972 (define_insn "mve_vdupq_m_n_f<mode>"
3973   [
3974    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3975         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3976                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3977                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3978          VDUPQ_M_N_F))
3979   ]
3980   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3981   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
3982   [(set_attr "type" "mve_move")
3983    (set_attr "length""8")])
3984
3985 ;;
3986 ;; [vfmaq_f])
3987 ;;
3988 (define_insn "mve_vfmaq_f<mode>"
3989   [
3990    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3991         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3992                        (match_operand:MVE_0 2 "s_register_operand" "w")
3993                        (match_operand:MVE_0 3 "s_register_operand" "w")]
3994          VFMAQ_F))
3995   ]
3996   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3997   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
3998   [(set_attr "type" "mve_move")
3999 ])
4000
4001 ;;
4002 ;; [vfmaq_n_f])
4003 ;;
4004 (define_insn "mve_vfmaq_n_f<mode>"
4005   [
4006    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4007         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4008                        (match_operand:MVE_0 2 "s_register_operand" "w")
4009                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4010          VFMAQ_N_F))
4011   ]
4012   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4013   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
4014   [(set_attr "type" "mve_move")
4015 ])
4016
4017 ;;
4018 ;; [vfmasq_n_f])
4019 ;;
4020 (define_insn "mve_vfmasq_n_f<mode>"
4021   [
4022    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4023         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4024                        (match_operand:MVE_0 2 "s_register_operand" "w")
4025                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4026          VFMASQ_N_F))
4027   ]
4028   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4029   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
4030   [(set_attr "type" "mve_move")
4031 ])
4032 ;;
4033 ;; [vfmsq_f])
4034 ;;
4035 (define_insn "mve_vfmsq_f<mode>"
4036   [
4037    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4038         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4039                        (match_operand:MVE_0 2 "s_register_operand" "w")
4040                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4041          VFMSQ_F))
4042   ]
4043   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4044   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
4045   [(set_attr "type" "mve_move")
4046 ])
4047
4048 ;;
4049 ;; [vmaxnmaq_m_f])
4050 ;;
4051 (define_insn "mve_vmaxnmaq_m_f<mode>"
4052   [
4053    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4054         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4055                        (match_operand:MVE_0 2 "s_register_operand" "w")
4056                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4057          VMAXNMAQ_M_F))
4058   ]
4059   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4060   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
4061   [(set_attr "type" "mve_move")
4062    (set_attr "length""8")])
4063 ;;
4064 ;; [vmaxnmavq_p_f])
4065 ;;
4066 (define_insn "mve_vmaxnmavq_p_f<mode>"
4067   [
4068    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4069         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4070                        (match_operand:MVE_0 2 "s_register_operand" "w")
4071                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4072          VMAXNMAVQ_P_F))
4073   ]
4074   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4075   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
4076   [(set_attr "type" "mve_move")
4077    (set_attr "length""8")])
4078
4079 ;;
4080 ;; [vmaxnmvq_p_f])
4081 ;;
4082 (define_insn "mve_vmaxnmvq_p_f<mode>"
4083   [
4084    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4085         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4086                        (match_operand:MVE_0 2 "s_register_operand" "w")
4087                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4088          VMAXNMVQ_P_F))
4089   ]
4090   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4091   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
4092   [(set_attr "type" "mve_move")
4093    (set_attr "length""8")])
4094 ;;
4095 ;; [vminnmaq_m_f])
4096 ;;
4097 (define_insn "mve_vminnmaq_m_f<mode>"
4098   [
4099    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4100         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4101                        (match_operand:MVE_0 2 "s_register_operand" "w")
4102                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4103          VMINNMAQ_M_F))
4104   ]
4105   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4106   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
4107   [(set_attr "type" "mve_move")
4108    (set_attr "length""8")])
4109
4110 ;;
4111 ;; [vminnmavq_p_f])
4112 ;;
4113 (define_insn "mve_vminnmavq_p_f<mode>"
4114   [
4115    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4116         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4117                        (match_operand:MVE_0 2 "s_register_operand" "w")
4118                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4119          VMINNMAVQ_P_F))
4120   ]
4121   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4122   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
4123   [(set_attr "type" "mve_move")
4124    (set_attr "length""8")])
4125 ;;
4126 ;; [vminnmvq_p_f])
4127 ;;
4128 (define_insn "mve_vminnmvq_p_f<mode>"
4129   [
4130    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4131         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4132                        (match_operand:MVE_0 2 "s_register_operand" "w")
4133                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4134          VMINNMVQ_P_F))
4135   ]
4136   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4137   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
4138   [(set_attr "type" "mve_move")
4139    (set_attr "length""8")])
4140
4141 ;;
4142 ;; [vmlaldavaq_s, vmlaldavaq_u])
4143 ;;
4144 (define_insn "mve_vmlaldavaq_<supf><mode>"
4145   [
4146    (set (match_operand:DI 0 "s_register_operand" "=r")
4147         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4148                        (match_operand:MVE_5 2 "s_register_operand" "w")
4149                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4150          VMLALDAVAQ))
4151   ]
4152   "TARGET_HAVE_MVE"
4153   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4154   [(set_attr "type" "mve_move")
4155 ])
4156
4157 ;;
4158 ;; [vmlaldavaxq_s])
4159 ;;
4160 (define_insn "mve_vmlaldavaxq_s<mode>"
4161   [
4162    (set (match_operand:DI 0 "s_register_operand" "=r")
4163         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4164                        (match_operand:MVE_5 2 "s_register_operand" "w")
4165                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4166          VMLALDAVAXQ_S))
4167   ]
4168   "TARGET_HAVE_MVE"
4169   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4170   [(set_attr "type" "mve_move")
4171 ])
4172
4173 ;;
4174 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
4175 ;;
4176 (define_insn "mve_vmlaldavq_p_<supf><mode>"
4177   [
4178    (set (match_operand:DI 0 "s_register_operand" "=r")
4179         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4180                        (match_operand:MVE_5 2 "s_register_operand" "w")
4181                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4182          VMLALDAVQ_P))
4183   ]
4184   "TARGET_HAVE_MVE"
4185   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4186   [(set_attr "type" "mve_move")
4187    (set_attr "length""8")])
4188
4189 ;;
4190 ;; [vmlaldavxq_p_s])
4191 ;;
4192 (define_insn "mve_vmlaldavxq_p_s<mode>"
4193   [
4194    (set (match_operand:DI 0 "s_register_operand" "=r")
4195         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4196                        (match_operand:MVE_5 2 "s_register_operand" "w")
4197                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4198          VMLALDAVXQ_P_S))
4199   ]
4200   "TARGET_HAVE_MVE"
4201   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
4202   [(set_attr "type" "mve_move")
4203    (set_attr "length""8")])
4204 ;;
4205 ;; [vmlsldavaq_s])
4206 ;;
4207 (define_insn "mve_vmlsldavaq_s<mode>"
4208   [
4209    (set (match_operand:DI 0 "s_register_operand" "=r")
4210         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4211                        (match_operand:MVE_5 2 "s_register_operand" "w")
4212                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4213          VMLSLDAVAQ_S))
4214   ]
4215   "TARGET_HAVE_MVE"
4216   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4217   [(set_attr "type" "mve_move")
4218 ])
4219
4220 ;;
4221 ;; [vmlsldavaxq_s])
4222 ;;
4223 (define_insn "mve_vmlsldavaxq_s<mode>"
4224   [
4225    (set (match_operand:DI 0 "s_register_operand" "=r")
4226         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4227                        (match_operand:MVE_5 2 "s_register_operand" "w")
4228                        (match_operand:MVE_5 3 "s_register_operand" "w")]
4229          VMLSLDAVAXQ_S))
4230   ]
4231   "TARGET_HAVE_MVE"
4232   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
4233   [(set_attr "type" "mve_move")
4234 ])
4235
4236 ;;
4237 ;; [vmlsldavq_p_s])
4238 ;;
4239 (define_insn "mve_vmlsldavq_p_s<mode>"
4240   [
4241    (set (match_operand:DI 0 "s_register_operand" "=r")
4242         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4243                        (match_operand:MVE_5 2 "s_register_operand" "w")
4244                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4245          VMLSLDAVQ_P_S))
4246   ]
4247   "TARGET_HAVE_MVE"
4248   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4249   [(set_attr "type" "mve_move")
4250    (set_attr "length""8")])
4251
4252 ;;
4253 ;; [vmlsldavxq_p_s])
4254 ;;
4255 (define_insn "mve_vmlsldavxq_p_s<mode>"
4256   [
4257    (set (match_operand:DI 0 "s_register_operand" "=r")
4258         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
4259                        (match_operand:MVE_5 2 "s_register_operand" "w")
4260                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4261          VMLSLDAVXQ_P_S))
4262   ]
4263   "TARGET_HAVE_MVE"
4264   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
4265   [(set_attr "type" "mve_move")
4266    (set_attr "length""8")])
4267 ;;
4268 ;; [vmovlbq_m_u, vmovlbq_m_s])
4269 ;;
4270 (define_insn "mve_vmovlbq_m_<supf><mode>"
4271   [
4272    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4273         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4274                        (match_operand:MVE_3 2 "s_register_operand" "w")
4275                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4276          VMOVLBQ_M))
4277   ]
4278   "TARGET_HAVE_MVE"
4279   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
4280   [(set_attr "type" "mve_move")
4281    (set_attr "length""8")])
4282 ;;
4283 ;; [vmovltq_m_u, vmovltq_m_s])
4284 ;;
4285 (define_insn "mve_vmovltq_m_<supf><mode>"
4286   [
4287    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
4288         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
4289                        (match_operand:MVE_3 2 "s_register_operand" "w")
4290                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4291          VMOVLTQ_M))
4292   ]
4293   "TARGET_HAVE_MVE"
4294   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
4295   [(set_attr "type" "mve_move")
4296    (set_attr "length""8")])
4297 ;;
4298 ;; [vmovnbq_m_u, vmovnbq_m_s])
4299 ;;
4300 (define_insn "mve_vmovnbq_m_<supf><mode>"
4301   [
4302    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4303         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4304                        (match_operand:MVE_5 2 "s_register_operand" "w")
4305                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4306          VMOVNBQ_M))
4307   ]
4308   "TARGET_HAVE_MVE"
4309   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
4310   [(set_attr "type" "mve_move")
4311    (set_attr "length""8")])
4312
4313 ;;
4314 ;; [vmovntq_m_u, vmovntq_m_s])
4315 ;;
4316 (define_insn "mve_vmovntq_m_<supf><mode>"
4317   [
4318    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4319         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4320                        (match_operand:MVE_5 2 "s_register_operand" "w")
4321                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4322          VMOVNTQ_M))
4323   ]
4324   "TARGET_HAVE_MVE"
4325   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
4326   [(set_attr "type" "mve_move")
4327    (set_attr "length""8")])
4328
4329 ;;
4330 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
4331 ;;
4332 (define_insn "mve_vmvnq_m_n_<supf><mode>"
4333   [
4334    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4335         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4336                        (match_operand:SI 2 "immediate_operand" "i")
4337                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4338          VMVNQ_M_N))
4339   ]
4340   "TARGET_HAVE_MVE"
4341   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
4342   [(set_attr "type" "mve_move")
4343    (set_attr "length""8")])
4344 ;;
4345 ;; [vnegq_m_f])
4346 ;;
4347 (define_insn "mve_vnegq_m_f<mode>"
4348   [
4349    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4350         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4351                        (match_operand:MVE_0 2 "s_register_operand" "w")
4352                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4353          VNEGQ_M_F))
4354   ]
4355   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4356   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
4357   [(set_attr "type" "mve_move")
4358    (set_attr "length""8")])
4359
4360 ;;
4361 ;; [vorrq_m_n_s, vorrq_m_n_u])
4362 ;;
4363 (define_insn "mve_vorrq_m_n_<supf><mode>"
4364   [
4365    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4366         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4367                        (match_operand:SI 2 "immediate_operand" "i")
4368                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4369          VORRQ_M_N))
4370   ]
4371   "TARGET_HAVE_MVE"
4372   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
4373   [(set_attr "type" "mve_move")
4374    (set_attr "length""8")])
4375 ;;
4376 ;; [vpselq_f])
4377 ;;
4378 (define_insn "@mve_vpselq_f<mode>"
4379   [
4380    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4381         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
4382                        (match_operand:MVE_0 2 "s_register_operand" "w")
4383                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4384          VPSELQ_F))
4385   ]
4386   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4387   "vpsel %q0, %q1, %q2"
4388   [(set_attr "type" "mve_move")
4389 ])
4390
4391 ;;
4392 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
4393 ;;
4394 (define_insn "mve_vqmovnbq_m_<supf><mode>"
4395   [
4396    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4397         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4398                        (match_operand:MVE_5 2 "s_register_operand" "w")
4399                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4400          VQMOVNBQ_M))
4401   ]
4402   "TARGET_HAVE_MVE"
4403   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
4404   [(set_attr "type" "mve_move")
4405    (set_attr "length""8")])
4406
4407 ;;
4408 ;; [vqmovntq_m_u, vqmovntq_m_s])
4409 ;;
4410 (define_insn "mve_vqmovntq_m_<supf><mode>"
4411   [
4412    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4413         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4414                        (match_operand:MVE_5 2 "s_register_operand" "w")
4415                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4416          VQMOVNTQ_M))
4417   ]
4418   "TARGET_HAVE_MVE"
4419   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
4420   [(set_attr "type" "mve_move")
4421    (set_attr "length""8")])
4422
4423 ;;
4424 ;; [vqmovunbq_m_s])
4425 ;;
4426 (define_insn "mve_vqmovunbq_m_s<mode>"
4427   [
4428    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4429         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4430                        (match_operand:MVE_5 2 "s_register_operand" "w")
4431                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4432          VQMOVUNBQ_M_S))
4433   ]
4434   "TARGET_HAVE_MVE"
4435   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
4436   [(set_attr "type" "mve_move")
4437    (set_attr "length""8")])
4438
4439 ;;
4440 ;; [vqmovuntq_m_s])
4441 ;;
4442 (define_insn "mve_vqmovuntq_m_s<mode>"
4443   [
4444    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4445         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4446                        (match_operand:MVE_5 2 "s_register_operand" "w")
4447                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4448          VQMOVUNTQ_M_S))
4449   ]
4450   "TARGET_HAVE_MVE"
4451   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
4452   [(set_attr "type" "mve_move")
4453    (set_attr "length""8")])
4454
4455 ;;
4456 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
4457 ;;
4458 (define_insn "mve_vqrshrntq_n_<supf><mode>"
4459   [
4460    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4461         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4462                        (match_operand:MVE_5 2 "s_register_operand" "w")
4463                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4464          VQRSHRNTQ_N))
4465   ]
4466   "TARGET_HAVE_MVE"
4467   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
4468   [(set_attr "type" "mve_move")
4469 ])
4470
4471 ;;
4472 ;; [vqrshruntq_n_s])
4473 ;;
4474 (define_insn "mve_vqrshruntq_n_s<mode>"
4475   [
4476    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4477         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4478                        (match_operand:MVE_5 2 "s_register_operand" "w")
4479                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4480          VQRSHRUNTQ_N_S))
4481   ]
4482   "TARGET_HAVE_MVE"
4483   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
4484   [(set_attr "type" "mve_move")
4485 ])
4486
4487 ;;
4488 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
4489 ;;
4490 (define_insn "mve_vqshrnbq_n_<supf><mode>"
4491   [
4492    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4493         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4494                        (match_operand:MVE_5 2 "s_register_operand" "w")
4495                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4496          VQSHRNBQ_N))
4497   ]
4498   "TARGET_HAVE_MVE"
4499   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4500   [(set_attr "type" "mve_move")
4501 ])
4502
4503 ;;
4504 ;; [vqshrntq_n_u, vqshrntq_n_s])
4505 ;;
4506 (define_insn "mve_vqshrntq_n_<supf><mode>"
4507   [
4508    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4509         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4510                        (match_operand:MVE_5 2 "s_register_operand" "w")
4511                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4512          VQSHRNTQ_N))
4513   ]
4514   "TARGET_HAVE_MVE"
4515   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
4516   [(set_attr "type" "mve_move")
4517 ])
4518
4519 ;;
4520 ;; [vqshrunbq_n_s])
4521 ;;
4522 (define_insn "mve_vqshrunbq_n_s<mode>"
4523   [
4524    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4525         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4526                        (match_operand:MVE_5 2 "s_register_operand" "w")
4527                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4528          VQSHRUNBQ_N_S))
4529   ]
4530   "TARGET_HAVE_MVE"
4531   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
4532   [(set_attr "type" "mve_move")
4533 ])
4534
4535 ;;
4536 ;; [vqshruntq_n_s])
4537 ;;
4538 (define_insn "mve_vqshruntq_n_s<mode>"
4539   [
4540    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4541         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4542                        (match_operand:MVE_5 2 "s_register_operand" "w")
4543                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4544          VQSHRUNTQ_N_S))
4545   ]
4546   "TARGET_HAVE_MVE"
4547   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
4548   [(set_attr "type" "mve_move")
4549 ])
4550
4551 ;;
4552 ;; [vrev32q_m_f])
4553 ;;
4554 (define_insn "mve_vrev32q_m_fv8hf"
4555   [
4556    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4557         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4558                        (match_operand:V8HF 2 "s_register_operand" "w")
4559                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4560          VREV32Q_M_F))
4561   ]
4562   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4563   "vpst\;vrev32t.16 %q0, %q2"
4564   [(set_attr "type" "mve_move")
4565    (set_attr "length""8")])
4566
4567 ;;
4568 ;; [vrev32q_m_s, vrev32q_m_u])
4569 ;;
4570 (define_insn "mve_vrev32q_m_<supf><mode>"
4571   [
4572    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
4573         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
4574                        (match_operand:MVE_3 2 "s_register_operand" "w")
4575                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4576          VREV32Q_M))
4577   ]
4578   "TARGET_HAVE_MVE"
4579   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
4580   [(set_attr "type" "mve_move")
4581    (set_attr "length""8")])
4582
4583 ;;
4584 ;; [vrev64q_m_f])
4585 ;;
4586 (define_insn "mve_vrev64q_m_f<mode>"
4587   [
4588    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4589         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4590                        (match_operand:MVE_0 2 "s_register_operand" "w")
4591                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4592          VREV64Q_M_F))
4593   ]
4594   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4595   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
4596   [(set_attr "type" "mve_move")
4597    (set_attr "length""8")])
4598
4599 ;;
4600 ;; [vrmlaldavhaxq_s])
4601 ;;
4602 (define_insn "mve_vrmlaldavhaxq_sv4si"
4603   [
4604    (set (match_operand:DI 0 "s_register_operand" "=r")
4605         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4606                        (match_operand:V4SI 2 "s_register_operand" "w")
4607                        (match_operand:V4SI 3 "s_register_operand" "w")]
4608          VRMLALDAVHAXQ_S))
4609   ]
4610   "TARGET_HAVE_MVE"
4611   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
4612   [(set_attr "type" "mve_move")
4613 ])
4614
4615 ;;
4616 ;; [vrmlaldavhxq_p_s])
4617 ;;
4618 (define_insn "mve_vrmlaldavhxq_p_sv4si"
4619   [
4620    (set (match_operand:DI 0 "s_register_operand" "=r")
4621         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4622                        (match_operand:V4SI 2 "s_register_operand" "w")
4623                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4624          VRMLALDAVHXQ_P_S))
4625   ]
4626   "TARGET_HAVE_MVE"
4627   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
4628   [(set_attr "type" "mve_move")
4629    (set_attr "length""8")])
4630
4631 ;;
4632 ;; [vrmlsldavhaxq_s])
4633 ;;
4634 (define_insn "mve_vrmlsldavhaxq_sv4si"
4635   [
4636    (set (match_operand:DI 0 "s_register_operand" "=r")
4637         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4638                        (match_operand:V4SI 2 "s_register_operand" "w")
4639                        (match_operand:V4SI 3 "s_register_operand" "w")]
4640          VRMLSLDAVHAXQ_S))
4641   ]
4642   "TARGET_HAVE_MVE"
4643   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
4644   [(set_attr "type" "mve_move")
4645 ])
4646
4647 ;;
4648 ;; [vrmlsldavhq_p_s])
4649 ;;
4650 (define_insn "mve_vrmlsldavhq_p_sv4si"
4651   [
4652    (set (match_operand:DI 0 "s_register_operand" "=r")
4653         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4654                        (match_operand:V4SI 2 "s_register_operand" "w")
4655                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4656          VRMLSLDAVHQ_P_S))
4657   ]
4658   "TARGET_HAVE_MVE"
4659   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
4660   [(set_attr "type" "mve_move")
4661    (set_attr "length""8")])
4662
4663 ;;
4664 ;; [vrmlsldavhxq_p_s])
4665 ;;
4666 (define_insn "mve_vrmlsldavhxq_p_sv4si"
4667   [
4668    (set (match_operand:DI 0 "s_register_operand" "=r")
4669         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4670                        (match_operand:V4SI 2 "s_register_operand" "w")
4671                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4672          VRMLSLDAVHXQ_P_S))
4673   ]
4674   "TARGET_HAVE_MVE"
4675   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
4676   [(set_attr "type" "mve_move")
4677    (set_attr "length""8")])
4678
4679 ;;
4680 ;; [vrndaq_m_f])
4681 ;;
4682 (define_insn "mve_vrndaq_m_f<mode>"
4683   [
4684    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4685         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4686                        (match_operand:MVE_0 2 "s_register_operand" "w")
4687                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4688          VRNDAQ_M_F))
4689   ]
4690   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4691   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
4692   [(set_attr "type" "mve_move")
4693    (set_attr "length""8")])
4694
4695 ;;
4696 ;; [vrndmq_m_f])
4697 ;;
4698 (define_insn "mve_vrndmq_m_f<mode>"
4699   [
4700    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4701         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4702                        (match_operand:MVE_0 2 "s_register_operand" "w")
4703                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4704          VRNDMQ_M_F))
4705   ]
4706   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4707   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
4708   [(set_attr "type" "mve_move")
4709    (set_attr "length""8")])
4710
4711 ;;
4712 ;; [vrndnq_m_f])
4713 ;;
4714 (define_insn "mve_vrndnq_m_f<mode>"
4715   [
4716    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4717         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4718                        (match_operand:MVE_0 2 "s_register_operand" "w")
4719                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4720          VRNDNQ_M_F))
4721   ]
4722   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4723   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
4724   [(set_attr "type" "mve_move")
4725    (set_attr "length""8")])
4726
4727 ;;
4728 ;; [vrndpq_m_f])
4729 ;;
4730 (define_insn "mve_vrndpq_m_f<mode>"
4731   [
4732    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4733         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4734                        (match_operand:MVE_0 2 "s_register_operand" "w")
4735                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4736          VRNDPQ_M_F))
4737   ]
4738   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4739   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
4740   [(set_attr "type" "mve_move")
4741    (set_attr "length""8")])
4742
4743 ;;
4744 ;; [vrndxq_m_f])
4745 ;;
4746 (define_insn "mve_vrndxq_m_f<mode>"
4747   [
4748    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4749         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4750                        (match_operand:MVE_0 2 "s_register_operand" "w")
4751                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4752          VRNDXQ_M_F))
4753   ]
4754   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4755   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
4756   [(set_attr "type" "mve_move")
4757    (set_attr "length""8")])
4758
4759 ;;
4760 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
4761 ;;
4762 (define_insn "mve_vrshrnbq_n_<supf><mode>"
4763   [
4764    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4765         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4766                        (match_operand:MVE_5 2 "s_register_operand" "w")
4767                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4768          VRSHRNBQ_N))
4769   ]
4770   "TARGET_HAVE_MVE"
4771   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
4772   [(set_attr "type" "mve_move")
4773 ])
4774
4775 ;;
4776 ;; [vrshrntq_n_u, vrshrntq_n_s])
4777 ;;
4778 (define_insn "mve_vrshrntq_n_<supf><mode>"
4779   [
4780    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4781         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4782                        (match_operand:MVE_5 2 "s_register_operand" "w")
4783                        (match_operand:SI 3 "mve_imm_8" "Rb")]
4784          VRSHRNTQ_N))
4785   ]
4786   "TARGET_HAVE_MVE"
4787   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
4788   [(set_attr "type" "mve_move")
4789 ])
4790
4791 ;;
4792 ;; [vshrnbq_n_u, vshrnbq_n_s])
4793 ;;
4794 (define_insn "mve_vshrnbq_n_<supf><mode>"
4795   [
4796    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4797         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4798                        (match_operand:MVE_5 2 "s_register_operand" "w")
4799                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4800          VSHRNBQ_N))
4801   ]
4802   "TARGET_HAVE_MVE"
4803   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
4804   [(set_attr "type" "mve_move")
4805 ])
4806
4807 ;;
4808 ;; [vshrntq_n_s, vshrntq_n_u])
4809 ;;
4810 (define_insn "mve_vshrntq_n_<supf><mode>"
4811   [
4812    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
4813         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
4814                                  (match_operand:MVE_5 2 "s_register_operand" "w")
4815                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
4816          VSHRNTQ_N))
4817   ]
4818   "TARGET_HAVE_MVE"
4819   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
4820   [(set_attr "type" "mve_move")
4821 ])
4822
4823 ;;
4824 ;; [vcvtmq_m_s, vcvtmq_m_u])
4825 ;;
4826 (define_insn "mve_vcvtmq_m_<supf><mode>"
4827   [
4828    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4829         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4830                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4831                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4832          VCVTMQ_M))
4833   ]
4834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4835   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4836   [(set_attr "type" "mve_move")
4837    (set_attr "length""8")])
4838
4839 ;;
4840 ;; [vcvtpq_m_u, vcvtpq_m_s])
4841 ;;
4842 (define_insn "mve_vcvtpq_m_<supf><mode>"
4843   [
4844    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4845         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4846                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4847                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4848          VCVTPQ_M))
4849   ]
4850   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4851   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4852   [(set_attr "type" "mve_move")
4853    (set_attr "length""8")])
4854
4855 ;;
4856 ;; [vcvtnq_m_s, vcvtnq_m_u])
4857 ;;
4858 (define_insn "mve_vcvtnq_m_<supf><mode>"
4859   [
4860    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4861         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4862                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4863                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4864          VCVTNQ_M))
4865   ]
4866   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4867   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4868   [(set_attr "type" "mve_move")
4869    (set_attr "length""8")])
4870
4871 ;;
4872 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
4873 ;;
4874 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
4875   [
4876    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4877         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4878                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4879                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
4880                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4881          VCVTQ_M_N_FROM_F))
4882   ]
4883   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4884   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
4885   [(set_attr "type" "mve_move")
4886    (set_attr "length""8")])
4887
4888 ;;
4889 ;; [vrev16q_m_u, vrev16q_m_s])
4890 ;;
4891 (define_insn "mve_vrev16q_m_<supf>v16qi"
4892   [
4893    (set (match_operand:V16QI 0 "s_register_operand" "=w")
4894         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
4895                        (match_operand:V16QI 2 "s_register_operand" "w")
4896                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4897          VREV16Q_M))
4898   ]
4899   "TARGET_HAVE_MVE"
4900   "vpst\;vrev16t.8 %q0, %q2"
4901   [(set_attr "type" "mve_move")
4902    (set_attr "length""8")])
4903
4904 ;;
4905 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
4906 ;;
4907 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
4908   [
4909    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
4910         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
4911                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
4912                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4913          VCVTQ_M_FROM_F))
4914   ]
4915   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4916   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
4917   [(set_attr "type" "mve_move")
4918    (set_attr "length""8")])
4919
4920 ;;
4921 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
4922 ;;
4923 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
4924   [
4925    (set (match_operand:DI 0 "s_register_operand" "=r")
4926         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
4927                     (match_operand:V4SI 2 "s_register_operand" "w")
4928                     (match_operand:HI 3 "vpr_register_operand" "Up")]
4929          VRMLALDAVHQ_P))
4930   ]
4931   "TARGET_HAVE_MVE"
4932   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
4933   [(set_attr "type" "mve_move")
4934    (set_attr "length""8")])
4935
4936 ;;
4937 ;; [vrmlsldavhaq_s])
4938 ;;
4939 (define_insn "mve_vrmlsldavhaq_sv4si"
4940   [
4941    (set (match_operand:DI 0 "s_register_operand" "=r")
4942         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4943                     (match_operand:V4SI 2 "s_register_operand" "w")
4944                     (match_operand:V4SI 3 "s_register_operand" "w")]
4945          VRMLSLDAVHAQ_S))
4946   ]
4947   "TARGET_HAVE_MVE"
4948   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
4949   [(set_attr "type" "mve_move")
4950 ])
4951
4952 ;;
4953 ;; [vabavq_p_s, vabavq_p_u])
4954 ;;
4955 (define_insn "mve_vabavq_p_<supf><mode>"
4956   [
4957    (set (match_operand:SI 0 "s_register_operand" "=r")
4958         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4959                     (match_operand:MVE_2 2 "s_register_operand" "w")
4960                     (match_operand:MVE_2 3 "s_register_operand" "w")
4961                     (match_operand:HI 4 "vpr_register_operand" "Up")]
4962          VABAVQ_P))
4963   ]
4964   "TARGET_HAVE_MVE"
4965   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
4966   [(set_attr "type" "mve_move")
4967 ])
4968
4969 ;;
4970 ;; [vqshluq_m_n_s])
4971 ;;
4972 (define_insn "mve_vqshluq_m_n_s<mode>"
4973   [
4974    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4975         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4976                        (match_operand:MVE_2 2 "s_register_operand" "w")
4977                        (match_operand:SI 3 "mve_imm_7" "Ra")
4978                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4979          VQSHLUQ_M_N_S))
4980   ]
4981   "TARGET_HAVE_MVE"
4982   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
4983   [(set_attr "type" "mve_move")])
4984
4985 ;;
4986 ;; [vshlq_m_s, vshlq_m_u])
4987 ;;
4988 (define_insn "mve_vshlq_m_<supf><mode>"
4989   [
4990    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4991         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4992                        (match_operand:MVE_2 2 "s_register_operand" "w")
4993                        (match_operand:MVE_2 3 "s_register_operand" "w")
4994                        (match_operand:HI 4 "vpr_register_operand" "Up")]
4995          VSHLQ_M))
4996   ]
4997   "TARGET_HAVE_MVE"
4998   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
4999   [(set_attr "type" "mve_move")])
5000
5001 ;;
5002 ;; [vsriq_m_n_s, vsriq_m_n_u])
5003 ;;
5004 (define_insn "mve_vsriq_m_n_<supf><mode>"
5005   [
5006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5008                        (match_operand:MVE_2 2 "s_register_operand" "w")
5009                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
5010                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5011          VSRIQ_M_N))
5012   ]
5013   "TARGET_HAVE_MVE"
5014   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
5015   [(set_attr "type" "mve_move")])
5016
5017 ;;
5018 ;; [vsubq_m_u, vsubq_m_s])
5019 ;;
5020 (define_insn "mve_vsubq_m_<supf><mode>"
5021   [
5022    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5023         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5024                        (match_operand:MVE_2 2 "s_register_operand" "w")
5025                        (match_operand:MVE_2 3 "s_register_operand" "w")
5026                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5027          VSUBQ_M))
5028   ]
5029   "TARGET_HAVE_MVE"
5030   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
5031   [(set_attr "type" "mve_move")])
5032
5033 ;;
5034 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
5035 ;;
5036 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
5037   [
5038    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5039         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5040                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5041                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5042                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5043          VCVTQ_M_N_TO_F))
5044   ]
5045   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5046   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5047   [(set_attr "type" "mve_move")
5048    (set_attr "length""8")])
5049 ;;
5050 ;; [vabdq_m_s, vabdq_m_u])
5051 ;;
5052 (define_insn "mve_vabdq_m_<supf><mode>"
5053   [
5054    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5055         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5056                        (match_operand:MVE_2 2 "s_register_operand" "w")
5057                        (match_operand:MVE_2 3 "s_register_operand" "w")
5058                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5059          VABDQ_M))
5060   ]
5061   "TARGET_HAVE_MVE"
5062   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5063   [(set_attr "type" "mve_move")
5064    (set_attr "length""8")])
5065
5066 ;;
5067 ;; [vaddq_m_n_s, vaddq_m_n_u])
5068 ;;
5069 (define_insn "mve_vaddq_m_n_<supf><mode>"
5070   [
5071    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5072         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5073                        (match_operand:MVE_2 2 "s_register_operand" "w")
5074                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5075                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5076          VADDQ_M_N))
5077   ]
5078   "TARGET_HAVE_MVE"
5079   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
5080   [(set_attr "type" "mve_move")
5081    (set_attr "length""8")])
5082
5083 ;;
5084 ;; [vaddq_m_u, vaddq_m_s])
5085 ;;
5086 (define_insn "mve_vaddq_m_<supf><mode>"
5087   [
5088    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5089         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5090                        (match_operand:MVE_2 2 "s_register_operand" "w")
5091                        (match_operand:MVE_2 3 "s_register_operand" "w")
5092                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5093          VADDQ_M))
5094   ]
5095   "TARGET_HAVE_MVE"
5096   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
5097   [(set_attr "type" "mve_move")
5098    (set_attr "length""8")])
5099
5100 ;;
5101 ;; [vandq_m_u, vandq_m_s])
5102 ;;
5103 (define_insn "mve_vandq_m_<supf><mode>"
5104   [
5105    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5106         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5107                        (match_operand:MVE_2 2 "s_register_operand" "w")
5108                        (match_operand:MVE_2 3 "s_register_operand" "w")
5109                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5110          VANDQ_M))
5111   ]
5112   "TARGET_HAVE_MVE"
5113   "vpst\;vandt %q0, %q2, %q3"
5114   [(set_attr "type" "mve_move")
5115    (set_attr "length""8")])
5116
5117 ;;
5118 ;; [vbicq_m_u, vbicq_m_s])
5119 ;;
5120 (define_insn "mve_vbicq_m_<supf><mode>"
5121   [
5122    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5123         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5124                        (match_operand:MVE_2 2 "s_register_operand" "w")
5125                        (match_operand:MVE_2 3 "s_register_operand" "w")
5126                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5127          VBICQ_M))
5128   ]
5129   "TARGET_HAVE_MVE"
5130   "vpst\;vbict %q0, %q2, %q3"
5131   [(set_attr "type" "mve_move")
5132    (set_attr "length""8")])
5133
5134 ;;
5135 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
5136 ;;
5137 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
5138   [
5139    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5140         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5141                        (match_operand:MVE_2 2 "s_register_operand" "w")
5142                        (match_operand:SI 3 "s_register_operand" "r")
5143                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5144          VBRSRQ_M_N))
5145   ]
5146   "TARGET_HAVE_MVE"
5147   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
5148   [(set_attr "type" "mve_move")
5149    (set_attr "length""8")])
5150
5151 ;;
5152 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
5153 ;;
5154 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
5155   [
5156    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5157         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5158                        (match_operand:MVE_2 2 "s_register_operand" "w")
5159                        (match_operand:MVE_2 3 "s_register_operand" "w")
5160                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5161          VCADDQ_ROT270_M))
5162   ]
5163   "TARGET_HAVE_MVE"
5164   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
5165   [(set_attr "type" "mve_move")
5166    (set_attr "length""8")])
5167
5168 ;;
5169 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
5170 ;;
5171 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
5172   [
5173    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5174         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5175                        (match_operand:MVE_2 2 "s_register_operand" "w")
5176                        (match_operand:MVE_2 3 "s_register_operand" "w")
5177                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5178          VCADDQ_ROT90_M))
5179   ]
5180   "TARGET_HAVE_MVE"
5181   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
5182   [(set_attr "type" "mve_move")
5183    (set_attr "length""8")])
5184
5185 ;;
5186 ;; [veorq_m_s, veorq_m_u])
5187 ;;
5188 (define_insn "mve_veorq_m_<supf><mode>"
5189   [
5190    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5191         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5192                        (match_operand:MVE_2 2 "s_register_operand" "w")
5193                        (match_operand:MVE_2 3 "s_register_operand" "w")
5194                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5195          VEORQ_M))
5196   ]
5197   "TARGET_HAVE_MVE"
5198   "vpst\;veort %q0, %q2, %q3"
5199   [(set_attr "type" "mve_move")
5200    (set_attr "length""8")])
5201
5202 ;;
5203 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
5204 ;;
5205 (define_insn "mve_vhaddq_m_n_<supf><mode>"
5206   [
5207    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5208         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5209                        (match_operand:MVE_2 2 "s_register_operand" "w")
5210                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5211                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5212          VHADDQ_M_N))
5213   ]
5214   "TARGET_HAVE_MVE"
5215   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5216   [(set_attr "type" "mve_move")
5217    (set_attr "length""8")])
5218
5219 ;;
5220 ;; [vhaddq_m_s, vhaddq_m_u])
5221 ;;
5222 (define_insn "mve_vhaddq_m_<supf><mode>"
5223   [
5224    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5225         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5226                        (match_operand:MVE_2 2 "s_register_operand" "w")
5227                        (match_operand:MVE_2 3 "s_register_operand" "w")
5228                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5229          VHADDQ_M))
5230   ]
5231   "TARGET_HAVE_MVE"
5232   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5233   [(set_attr "type" "mve_move")
5234    (set_attr "length""8")])
5235
5236 ;;
5237 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
5238 ;;
5239 (define_insn "mve_vhsubq_m_n_<supf><mode>"
5240   [
5241    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5242         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5243                        (match_operand:MVE_2 2 "s_register_operand" "w")
5244                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5245                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5246          VHSUBQ_M_N))
5247   ]
5248   "TARGET_HAVE_MVE"
5249   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5250   [(set_attr "type" "mve_move")
5251    (set_attr "length""8")])
5252
5253 ;;
5254 ;; [vhsubq_m_s, vhsubq_m_u])
5255 ;;
5256 (define_insn "mve_vhsubq_m_<supf><mode>"
5257   [
5258    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5259         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5260                        (match_operand:MVE_2 2 "s_register_operand" "w")
5261                        (match_operand:MVE_2 3 "s_register_operand" "w")
5262                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5263          VHSUBQ_M))
5264   ]
5265   "TARGET_HAVE_MVE"
5266   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5267   [(set_attr "type" "mve_move")
5268    (set_attr "length""8")])
5269
5270 ;;
5271 ;; [vmaxq_m_s, vmaxq_m_u])
5272 ;;
5273 (define_insn "mve_vmaxq_m_<supf><mode>"
5274   [
5275    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5276         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5277                        (match_operand:MVE_2 2 "s_register_operand" "w")
5278                        (match_operand:MVE_2 3 "s_register_operand" "w")
5279                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5280          VMAXQ_M))
5281   ]
5282   "TARGET_HAVE_MVE"
5283   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5284   [(set_attr "type" "mve_move")
5285    (set_attr "length""8")])
5286
5287 ;;
5288 ;; [vminq_m_s, vminq_m_u])
5289 ;;
5290 (define_insn "mve_vminq_m_<supf><mode>"
5291   [
5292    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5293         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5294                        (match_operand:MVE_2 2 "s_register_operand" "w")
5295                        (match_operand:MVE_2 3 "s_register_operand" "w")
5296                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5297          VMINQ_M))
5298   ]
5299   "TARGET_HAVE_MVE"
5300   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
5301   [(set_attr "type" "mve_move")
5302    (set_attr "length""8")])
5303
5304 ;;
5305 ;; [vmladavaq_p_u, vmladavaq_p_s])
5306 ;;
5307 (define_insn "mve_vmladavaq_p_<supf><mode>"
5308   [
5309    (set (match_operand:SI 0 "s_register_operand" "=Te")
5310         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5311                     (match_operand:MVE_2 2 "s_register_operand" "w")
5312                     (match_operand:MVE_2 3 "s_register_operand" "w")
5313                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5314          VMLADAVAQ_P))
5315   ]
5316   "TARGET_HAVE_MVE"
5317   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
5318   [(set_attr "type" "mve_move")
5319    (set_attr "length""8")])
5320
5321 ;;
5322 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
5323 ;;
5324 (define_insn "mve_vmlaq_m_n_<supf><mode>"
5325   [
5326    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5327         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5328                        (match_operand:MVE_2 2 "s_register_operand" "w")
5329                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5330                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5331          VMLAQ_M_N))
5332   ]
5333   "TARGET_HAVE_MVE"
5334   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
5335   [(set_attr "type" "mve_move")
5336    (set_attr "length""8")])
5337
5338 ;;
5339 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
5340 ;;
5341 (define_insn "mve_vmlasq_m_n_<supf><mode>"
5342   [
5343    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5344         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5345                        (match_operand:MVE_2 2 "s_register_operand" "w")
5346                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5347                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5348          VMLASQ_M_N))
5349   ]
5350   "TARGET_HAVE_MVE"
5351   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
5352   [(set_attr "type" "mve_move")
5353    (set_attr "length""8")])
5354
5355 ;;
5356 ;; [vmulhq_m_s, vmulhq_m_u])
5357 ;;
5358 (define_insn "mve_vmulhq_m_<supf><mode>"
5359   [
5360    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5361         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5362                        (match_operand:MVE_2 2 "s_register_operand" "w")
5363                        (match_operand:MVE_2 3 "s_register_operand" "w")
5364                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5365          VMULHQ_M))
5366   ]
5367   "TARGET_HAVE_MVE"
5368   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
5369   [(set_attr "type" "mve_move")
5370    (set_attr "length""8")])
5371
5372 ;;
5373 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
5374 ;;
5375 (define_insn "mve_vmullbq_int_m_<supf><mode>"
5376   [
5377    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5378         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5379                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5380                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5381                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5382          VMULLBQ_INT_M))
5383   ]
5384   "TARGET_HAVE_MVE"
5385   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5386   [(set_attr "type" "mve_move")
5387    (set_attr "length""8")])
5388
5389 ;;
5390 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
5391 ;;
5392 (define_insn "mve_vmulltq_int_m_<supf><mode>"
5393   [
5394    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
5395         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5396                                   (match_operand:MVE_2 2 "s_register_operand" "w")
5397                                   (match_operand:MVE_2 3 "s_register_operand" "w")
5398                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
5399          VMULLTQ_INT_M))
5400   ]
5401   "TARGET_HAVE_MVE"
5402   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
5403   [(set_attr "type" "mve_move")
5404    (set_attr "length""8")])
5405
5406 ;;
5407 ;; [vmulq_m_n_u, vmulq_m_n_s])
5408 ;;
5409 (define_insn "mve_vmulq_m_n_<supf><mode>"
5410   [
5411    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5412         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5413                        (match_operand:MVE_2 2 "s_register_operand" "w")
5414                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5415                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5416          VMULQ_M_N))
5417   ]
5418   "TARGET_HAVE_MVE"
5419   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
5420   [(set_attr "type" "mve_move")
5421    (set_attr "length""8")])
5422
5423 ;;
5424 ;; [vmulq_m_s, vmulq_m_u])
5425 ;;
5426 (define_insn "mve_vmulq_m_<supf><mode>"
5427   [
5428    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5429         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5430                        (match_operand:MVE_2 2 "s_register_operand" "w")
5431                        (match_operand:MVE_2 3 "s_register_operand" "w")
5432                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5433          VMULQ_M))
5434   ]
5435   "TARGET_HAVE_MVE"
5436   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
5437   [(set_attr "type" "mve_move")
5438    (set_attr "length""8")])
5439
5440 ;;
5441 ;; [vornq_m_u, vornq_m_s])
5442 ;;
5443 (define_insn "mve_vornq_m_<supf><mode>"
5444   [
5445    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5446         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5447                        (match_operand:MVE_2 2 "s_register_operand" "w")
5448                        (match_operand:MVE_2 3 "s_register_operand" "w")
5449                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5450          VORNQ_M))
5451   ]
5452   "TARGET_HAVE_MVE"
5453   "vpst\;vornt %q0, %q2, %q3"
5454   [(set_attr "type" "mve_move")
5455    (set_attr "length""8")])
5456
5457 ;;
5458 ;; [vorrq_m_s, vorrq_m_u])
5459 ;;
5460 (define_insn "mve_vorrq_m_<supf><mode>"
5461   [
5462    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5463         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5464                        (match_operand:MVE_2 2 "s_register_operand" "w")
5465                        (match_operand:MVE_2 3 "s_register_operand" "w")
5466                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5467          VORRQ_M))
5468   ]
5469   "TARGET_HAVE_MVE"
5470   "vpst\;vorrt %q0, %q2, %q3"
5471   [(set_attr "type" "mve_move")
5472    (set_attr "length""8")])
5473
5474 ;;
5475 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
5476 ;;
5477 (define_insn "mve_vqaddq_m_n_<supf><mode>"
5478   [
5479    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5480         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5481                        (match_operand:MVE_2 2 "s_register_operand" "w")
5482                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5483                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5484          VQADDQ_M_N))
5485   ]
5486   "TARGET_HAVE_MVE"
5487   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5488   [(set_attr "type" "mve_move")
5489    (set_attr "length""8")])
5490
5491 ;;
5492 ;; [vqaddq_m_u, vqaddq_m_s])
5493 ;;
5494 (define_insn "mve_vqaddq_m_<supf><mode>"
5495   [
5496    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5497         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5498                        (match_operand:MVE_2 2 "s_register_operand" "w")
5499                        (match_operand:MVE_2 3 "s_register_operand" "w")
5500                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5501          VQADDQ_M))
5502   ]
5503   "TARGET_HAVE_MVE"
5504   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5505   [(set_attr "type" "mve_move")
5506    (set_attr "length""8")])
5507
5508 ;;
5509 ;; [vqdmlahq_m_n_s])
5510 ;;
5511 (define_insn "mve_vqdmlahq_m_n_s<mode>"
5512   [
5513    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5514         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5515                        (match_operand:MVE_2 2 "s_register_operand" "w")
5516                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5517                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5518          VQDMLAHQ_M_N_S))
5519   ]
5520   "TARGET_HAVE_MVE"
5521   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5522   [(set_attr "type" "mve_move")
5523    (set_attr "length""8")])
5524
5525 ;;
5526 ;; [vqdmlashq_m_n_s])
5527 ;;
5528 (define_insn "mve_vqdmlashq_m_n_s<mode>"
5529   [
5530    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5531         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5532                        (match_operand:MVE_2 2 "s_register_operand" "w")
5533                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5534                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5535          VQDMLASHQ_M_N_S))
5536   ]
5537   "TARGET_HAVE_MVE"
5538   "vpst\;vqdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5539   [(set_attr "type" "mve_move")
5540    (set_attr "length""8")])
5541
5542 ;;
5543 ;; [vqrdmlahq_m_n_s])
5544 ;;
5545 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
5546   [
5547    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5548         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5549                        (match_operand:MVE_2 2 "s_register_operand" "w")
5550                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5551                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5552          VQRDMLAHQ_M_N_S))
5553   ]
5554   "TARGET_HAVE_MVE"
5555   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
5556   [(set_attr "type" "mve_move")
5557    (set_attr "length""8")])
5558
5559 ;;
5560 ;; [vqrdmlashq_m_n_s])
5561 ;;
5562 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
5563   [
5564    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5565         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5566                        (match_operand:MVE_2 2 "s_register_operand" "w")
5567                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5568                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5569          VQRDMLASHQ_M_N_S))
5570   ]
5571   "TARGET_HAVE_MVE"
5572   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
5573   [(set_attr "type" "mve_move")
5574    (set_attr "length""8")])
5575
5576 ;;
5577 ;; [vqrshlq_m_u, vqrshlq_m_s])
5578 ;;
5579 (define_insn "mve_vqrshlq_m_<supf><mode>"
5580   [
5581    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5582         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5583                        (match_operand:MVE_2 2 "s_register_operand" "w")
5584                        (match_operand:MVE_2 3 "s_register_operand" "w")
5585                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5586          VQRSHLQ_M))
5587   ]
5588   "TARGET_HAVE_MVE"
5589   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5590   [(set_attr "type" "mve_move")
5591    (set_attr "length""8")])
5592
5593 ;;
5594 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
5595 ;;
5596 (define_insn "mve_vqshlq_m_n_<supf><mode>"
5597   [
5598    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5599         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5600                        (match_operand:MVE_2 2 "s_register_operand" "w")
5601                        (match_operand:SI 3 "immediate_operand" "i")
5602                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5603          VQSHLQ_M_N))
5604   ]
5605   "TARGET_HAVE_MVE"
5606   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5607   [(set_attr "type" "mve_move")
5608    (set_attr "length""8")])
5609
5610 ;;
5611 ;; [vqshlq_m_u, vqshlq_m_s])
5612 ;;
5613 (define_insn "mve_vqshlq_m_<supf><mode>"
5614   [
5615    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5616         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5617                        (match_operand:MVE_2 2 "s_register_operand" "w")
5618                        (match_operand:MVE_2 3 "s_register_operand" "w")
5619                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5620          VQSHLQ_M))
5621   ]
5622   "TARGET_HAVE_MVE"
5623   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5624   [(set_attr "type" "mve_move")
5625    (set_attr "length""8")])
5626
5627 ;;
5628 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
5629 ;;
5630 (define_insn "mve_vqsubq_m_n_<supf><mode>"
5631   [
5632    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5633         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5634                        (match_operand:MVE_2 2 "s_register_operand" "w")
5635                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5636                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5637          VQSUBQ_M_N))
5638   ]
5639   "TARGET_HAVE_MVE"
5640   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5641   [(set_attr "type" "mve_move")
5642    (set_attr "length""8")])
5643
5644 ;;
5645 ;; [vqsubq_m_u, vqsubq_m_s])
5646 ;;
5647 (define_insn "mve_vqsubq_m_<supf><mode>"
5648   [
5649    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5650         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5651                        (match_operand:MVE_2 2 "s_register_operand" "w")
5652                        (match_operand:MVE_2 3 "s_register_operand" "w")
5653                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5654          VQSUBQ_M))
5655   ]
5656   "TARGET_HAVE_MVE"
5657   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5658   [(set_attr "type" "mve_move")
5659    (set_attr "length""8")])
5660
5661 ;;
5662 ;; [vrhaddq_m_u, vrhaddq_m_s])
5663 ;;
5664 (define_insn "mve_vrhaddq_m_<supf><mode>"
5665   [
5666    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5667         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5668                        (match_operand:MVE_2 2 "s_register_operand" "w")
5669                        (match_operand:MVE_2 3 "s_register_operand" "w")
5670                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5671          VRHADDQ_M))
5672   ]
5673   "TARGET_HAVE_MVE"
5674   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5675   [(set_attr "type" "mve_move")
5676    (set_attr "length""8")])
5677
5678 ;;
5679 ;; [vrmulhq_m_u, vrmulhq_m_s])
5680 ;;
5681 (define_insn "mve_vrmulhq_m_<supf><mode>"
5682   [
5683    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5684         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5685                        (match_operand:MVE_2 2 "s_register_operand" "w")
5686                        (match_operand:MVE_2 3 "s_register_operand" "w")
5687                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5688          VRMULHQ_M))
5689   ]
5690   "TARGET_HAVE_MVE"
5691   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5692   [(set_attr "type" "mve_move")
5693    (set_attr "length""8")])
5694
5695 ;;
5696 ;; [vrshlq_m_s, vrshlq_m_u])
5697 ;;
5698 (define_insn "mve_vrshlq_m_<supf><mode>"
5699   [
5700    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5701         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5702                        (match_operand:MVE_2 2 "s_register_operand" "w")
5703                        (match_operand:MVE_2 3 "s_register_operand" "w")
5704                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5705          VRSHLQ_M))
5706   ]
5707   "TARGET_HAVE_MVE"
5708   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5709   [(set_attr "type" "mve_move")
5710    (set_attr "length""8")])
5711
5712 ;;
5713 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
5714 ;;
5715 (define_insn "mve_vrshrq_m_n_<supf><mode>"
5716   [
5717    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5718         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5719                        (match_operand:MVE_2 2 "s_register_operand" "w")
5720                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5721                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5722          VRSHRQ_M_N))
5723   ]
5724   "TARGET_HAVE_MVE"
5725   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5726   [(set_attr "type" "mve_move")
5727    (set_attr "length""8")])
5728
5729 ;;
5730 ;; [vshlq_m_n_s, vshlq_m_n_u])
5731 ;;
5732 (define_insn "mve_vshlq_m_n_<supf><mode>"
5733   [
5734    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5735         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5736                        (match_operand:MVE_2 2 "s_register_operand" "w")
5737                        (match_operand:SI 3 "immediate_operand" "i")
5738                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5739          VSHLQ_M_N))
5740   ]
5741   "TARGET_HAVE_MVE"
5742   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5743   [(set_attr "type" "mve_move")
5744    (set_attr "length""8")])
5745
5746 ;;
5747 ;; [vshrq_m_n_s, vshrq_m_n_u])
5748 ;;
5749 (define_insn "mve_vshrq_m_n_<supf><mode>"
5750   [
5751    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5752         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5753                        (match_operand:MVE_2 2 "s_register_operand" "w")
5754                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5755                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5756          VSHRQ_M_N))
5757   ]
5758   "TARGET_HAVE_MVE"
5759   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5760   [(set_attr "type" "mve_move")
5761    (set_attr "length""8")])
5762
5763 ;;
5764 ;; [vsliq_m_n_u, vsliq_m_n_s])
5765 ;;
5766 (define_insn "mve_vsliq_m_n_<supf><mode>"
5767    [
5768    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5769        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5770                        (match_operand:MVE_2 2 "s_register_operand" "w")
5771                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
5772                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5773          VSLIQ_M_N))
5774   ]
5775   "TARGET_HAVE_MVE"
5776   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
5777   [(set_attr "type" "mve_move")
5778    (set_attr "length""8")])
5779
5780 ;;
5781 ;; [vsubq_m_n_s, vsubq_m_n_u])
5782 ;;
5783 (define_insn "mve_vsubq_m_n_<supf><mode>"
5784   [
5785    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5786         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5787                        (match_operand:MVE_2 2 "s_register_operand" "w")
5788                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5789                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5790          VSUBQ_M_N))
5791   ]
5792   "TARGET_HAVE_MVE"
5793   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
5794   [(set_attr "type" "mve_move")
5795    (set_attr "length""8")])
5796
5797 ;;
5798 ;; [vhcaddq_rot270_m_s])
5799 ;;
5800 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
5801   [
5802    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5803         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5804                        (match_operand:MVE_2 2 "s_register_operand" "w")
5805                        (match_operand:MVE_2 3 "s_register_operand" "w")
5806                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5807          VHCADDQ_ROT270_M_S))
5808   ]
5809   "TARGET_HAVE_MVE"
5810   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
5811   [(set_attr "type" "mve_move")
5812    (set_attr "length""8")])
5813
5814 ;;
5815 ;; [vhcaddq_rot90_m_s])
5816 ;;
5817 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
5818   [
5819    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
5820         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5821                        (match_operand:MVE_2 2 "s_register_operand" "w")
5822                        (match_operand:MVE_2 3 "s_register_operand" "w")
5823                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5824          VHCADDQ_ROT90_M_S))
5825   ]
5826   "TARGET_HAVE_MVE"
5827   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
5828   [(set_attr "type" "mve_move")
5829    (set_attr "length""8")])
5830
5831 ;;
5832 ;; [vmladavaxq_p_s])
5833 ;;
5834 (define_insn "mve_vmladavaxq_p_s<mode>"
5835   [
5836    (set (match_operand:SI 0 "s_register_operand" "=Te")
5837         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5838                        (match_operand:MVE_2 2 "s_register_operand" "w")
5839                        (match_operand:MVE_2 3 "s_register_operand" "w")
5840                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5841          VMLADAVAXQ_P_S))
5842   ]
5843   "TARGET_HAVE_MVE"
5844   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5845   [(set_attr "type" "mve_move")
5846    (set_attr "length""8")])
5847
5848 ;;
5849 ;; [vmlsdavaq_p_s])
5850 ;;
5851 (define_insn "mve_vmlsdavaq_p_s<mode>"
5852   [
5853    (set (match_operand:SI 0 "s_register_operand" "=Te")
5854         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5855                        (match_operand:MVE_2 2 "s_register_operand" "w")
5856                        (match_operand:MVE_2 3 "s_register_operand" "w")
5857                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5858          VMLSDAVAQ_P_S))
5859   ]
5860   "TARGET_HAVE_MVE"
5861   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
5862   [(set_attr "type" "mve_move")
5863    (set_attr "length""8")])
5864
5865 ;;
5866 ;; [vmlsdavaxq_p_s])
5867 ;;
5868 (define_insn "mve_vmlsdavaxq_p_s<mode>"
5869   [
5870    (set (match_operand:SI 0 "s_register_operand" "=Te")
5871         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5872                        (match_operand:MVE_2 2 "s_register_operand" "w")
5873                        (match_operand:MVE_2 3 "s_register_operand" "w")
5874                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5875          VMLSDAVAXQ_P_S))
5876   ]
5877   "TARGET_HAVE_MVE"
5878   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
5879   [(set_attr "type" "mve_move")
5880    (set_attr "length""8")])
5881
5882 ;;
5883 ;; [vqdmladhq_m_s])
5884 ;;
5885 (define_insn "mve_vqdmladhq_m_s<mode>"
5886   [
5887    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5888         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5889                        (match_operand:MVE_2 2 "s_register_operand" "w")
5890                        (match_operand:MVE_2 3 "s_register_operand" "w")
5891                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5892          VQDMLADHQ_M_S))
5893   ]
5894   "TARGET_HAVE_MVE"
5895   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5896   [(set_attr "type" "mve_move")
5897    (set_attr "length""8")])
5898
5899 ;;
5900 ;; [vqdmladhxq_m_s])
5901 ;;
5902 (define_insn "mve_vqdmladhxq_m_s<mode>"
5903   [
5904    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5905         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5906                        (match_operand:MVE_2 2 "s_register_operand" "w")
5907                        (match_operand:MVE_2 3 "s_register_operand" "w")
5908                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5909          VQDMLADHXQ_M_S))
5910   ]
5911   "TARGET_HAVE_MVE"
5912   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5913   [(set_attr "type" "mve_move")
5914    (set_attr "length""8")])
5915
5916 ;;
5917 ;; [vqdmlsdhq_m_s])
5918 ;;
5919 (define_insn "mve_vqdmlsdhq_m_s<mode>"
5920   [
5921    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5922         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5923                        (match_operand:MVE_2 2 "s_register_operand" "w")
5924                        (match_operand:MVE_2 3 "s_register_operand" "w")
5925                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5926          VQDMLSDHQ_M_S))
5927   ]
5928   "TARGET_HAVE_MVE"
5929   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5930   [(set_attr "type" "mve_move")
5931    (set_attr "length""8")])
5932
5933 ;;
5934 ;; [vqdmlsdhxq_m_s])
5935 ;;
5936 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
5937   [
5938    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5939         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5940                        (match_operand:MVE_2 2 "s_register_operand" "w")
5941                        (match_operand:MVE_2 3 "s_register_operand" "w")
5942                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5943          VQDMLSDHXQ_M_S))
5944   ]
5945   "TARGET_HAVE_MVE"
5946   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
5947   [(set_attr "type" "mve_move")
5948    (set_attr "length""8")])
5949
5950 ;;
5951 ;; [vqdmulhq_m_n_s])
5952 ;;
5953 (define_insn "mve_vqdmulhq_m_n_s<mode>"
5954   [
5955    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5956         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5957                        (match_operand:MVE_2 2 "s_register_operand" "w")
5958                        (match_operand:<V_elem> 3 "s_register_operand" "r")
5959                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5960          VQDMULHQ_M_N_S))
5961   ]
5962   "TARGET_HAVE_MVE"
5963   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
5964   [(set_attr "type" "mve_move")
5965    (set_attr "length""8")])
5966
5967 ;;
5968 ;; [vqdmulhq_m_s])
5969 ;;
5970 (define_insn "mve_vqdmulhq_m_s<mode>"
5971   [
5972    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5973         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5974                        (match_operand:MVE_2 2 "s_register_operand" "w")
5975                        (match_operand:MVE_2 3 "s_register_operand" "w")
5976                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5977          VQDMULHQ_M_S))
5978   ]
5979   "TARGET_HAVE_MVE"
5980   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5981   [(set_attr "type" "mve_move")
5982    (set_attr "length""8")])
5983
5984 ;;
5985 ;; [vqrdmladhq_m_s])
5986 ;;
5987 (define_insn "mve_vqrdmladhq_m_s<mode>"
5988   [
5989    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5990         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5991                        (match_operand:MVE_2 2 "s_register_operand" "w")
5992                        (match_operand:MVE_2 3 "s_register_operand" "w")
5993                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5994          VQRDMLADHQ_M_S))
5995   ]
5996   "TARGET_HAVE_MVE"
5997   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
5998   [(set_attr "type" "mve_move")
5999    (set_attr "length""8")])
6000
6001 ;;
6002 ;; [vqrdmladhxq_m_s])
6003 ;;
6004 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6005   [
6006    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6007         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6008                        (match_operand:MVE_2 2 "s_register_operand" "w")
6009                        (match_operand:MVE_2 3 "s_register_operand" "w")
6010                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6011          VQRDMLADHXQ_M_S))
6012   ]
6013   "TARGET_HAVE_MVE"
6014   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6015   [(set_attr "type" "mve_move")
6016    (set_attr "length""8")])
6017
6018 ;;
6019 ;; [vqrdmlsdhq_m_s])
6020 ;;
6021 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6022   [
6023    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6024         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6025                        (match_operand:MVE_2 2 "s_register_operand" "w")
6026                        (match_operand:MVE_2 3 "s_register_operand" "w")
6027                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6028          VQRDMLSDHQ_M_S))
6029   ]
6030   "TARGET_HAVE_MVE"
6031   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6032   [(set_attr "type" "mve_move")
6033    (set_attr "length""8")])
6034
6035 ;;
6036 ;; [vqrdmlsdhxq_m_s])
6037 ;;
6038 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
6039   [
6040    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6041         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6042                        (match_operand:MVE_2 2 "s_register_operand" "w")
6043                        (match_operand:MVE_2 3 "s_register_operand" "w")
6044                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6045          VQRDMLSDHXQ_M_S))
6046   ]
6047   "TARGET_HAVE_MVE"
6048   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6049   [(set_attr "type" "mve_move")
6050    (set_attr "length""8")])
6051
6052 ;;
6053 ;; [vqrdmulhq_m_n_s])
6054 ;;
6055 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
6056   [
6057    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6058         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6059                        (match_operand:MVE_2 2 "s_register_operand" "w")
6060                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6061                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6062          VQRDMULHQ_M_N_S))
6063   ]
6064   "TARGET_HAVE_MVE"
6065   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6066   [(set_attr "type" "mve_move")
6067    (set_attr "length""8")])
6068
6069 ;;
6070 ;; [vqrdmulhq_m_s])
6071 ;;
6072 (define_insn "mve_vqrdmulhq_m_s<mode>"
6073   [
6074    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6075         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6076                        (match_operand:MVE_2 2 "s_register_operand" "w")
6077                        (match_operand:MVE_2 3 "s_register_operand" "w")
6078                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6079          VQRDMULHQ_M_S))
6080   ]
6081   "TARGET_HAVE_MVE"
6082   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6083   [(set_attr "type" "mve_move")
6084    (set_attr "length""8")])
6085
6086 ;;
6087 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
6088 ;;
6089 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
6090   [
6091    (set (match_operand:DI 0 "s_register_operand" "=r")
6092         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6093                        (match_operand:MVE_5 2 "s_register_operand" "w")
6094                        (match_operand:MVE_5 3 "s_register_operand" "w")
6095                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6096          VMLALDAVAQ_P))
6097   ]
6098   "TARGET_HAVE_MVE"
6099   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6100   [(set_attr "type" "mve_move")
6101    (set_attr "length""8")])
6102
6103 ;;
6104 ;; [vmlaldavaxq_p_s])
6105 ;;
6106 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
6107   [
6108    (set (match_operand:DI 0 "s_register_operand" "=r")
6109         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6110                        (match_operand:MVE_5 2 "s_register_operand" "w")
6111                        (match_operand:MVE_5 3 "s_register_operand" "w")
6112                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6113          VMLALDAVAXQ_P))
6114   ]
6115   "TARGET_HAVE_MVE"
6116   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
6117   [(set_attr "type" "mve_move")
6118    (set_attr "length""8")])
6119
6120 ;;
6121 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
6122 ;;
6123 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
6124   [
6125    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6126         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6127                        (match_operand:MVE_5 2 "s_register_operand" "w")
6128                        (match_operand:SI 3 "mve_imm_8" "Rb")
6129                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6130          VQRSHRNBQ_M_N))
6131   ]
6132   "TARGET_HAVE_MVE"
6133   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6134   [(set_attr "type" "mve_move")
6135    (set_attr "length""8")])
6136
6137 ;;
6138 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
6139 ;;
6140 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
6141   [
6142    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6143         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6144                        (match_operand:MVE_5 2 "s_register_operand" "w")
6145                        (match_operand:SI 3 "mve_imm_8" "Rb")
6146                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6147          VQRSHRNTQ_M_N))
6148   ]
6149   "TARGET_HAVE_MVE"
6150   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
6151   [(set_attr "type" "mve_move")
6152    (set_attr "length""8")])
6153
6154 ;;
6155 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
6156 ;;
6157 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
6158   [
6159    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6160         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6161                        (match_operand:MVE_5 2 "s_register_operand" "w")
6162                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6163                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6164          VQSHRNBQ_M_N))
6165   ]
6166   "TARGET_HAVE_MVE"
6167   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6168   [(set_attr "type" "mve_move")
6169    (set_attr "length""8")])
6170
6171 ;;
6172 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
6173 ;;
6174 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
6175   [
6176    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6177         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6178                        (match_operand:MVE_5 2 "s_register_operand" "w")
6179                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6180                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6181          VQSHRNTQ_M_N))
6182   ]
6183   "TARGET_HAVE_MVE"
6184   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6185   [(set_attr "type" "mve_move")
6186    (set_attr "length""8")])
6187
6188 ;;
6189 ;; [vrmlaldavhaq_p_s])
6190 ;;
6191 (define_insn "mve_vrmlaldavhaq_p_sv4si"
6192   [
6193    (set (match_operand:DI 0 "s_register_operand" "=r")
6194         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6195                        (match_operand:V4SI 2 "s_register_operand" "w")
6196                        (match_operand:V4SI 3 "s_register_operand" "w")
6197                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6198          VRMLALDAVHAQ_P_S))
6199   ]
6200   "TARGET_HAVE_MVE"
6201   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
6202   [(set_attr "type" "mve_move")
6203    (set_attr "length""8")])
6204
6205 ;;
6206 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
6207 ;;
6208 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
6209   [
6210    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6211         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6212                        (match_operand:MVE_5 2 "s_register_operand" "w")
6213                        (match_operand:SI 3 "mve_imm_8" "Rb")
6214                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6215          VRSHRNBQ_M_N))
6216   ]
6217   "TARGET_HAVE_MVE"
6218   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6219   [(set_attr "type" "mve_move")
6220    (set_attr "length""8")])
6221
6222 ;;
6223 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
6224 ;;
6225 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
6226   [
6227    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6228         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6229                        (match_operand:MVE_5 2 "s_register_operand" "w")
6230                        (match_operand:SI 3 "mve_imm_8" "Rb")
6231                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6232          VRSHRNTQ_M_N))
6233   ]
6234   "TARGET_HAVE_MVE"
6235   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6236   [(set_attr "type" "mve_move")
6237    (set_attr "length""8")])
6238
6239 ;;
6240 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
6241 ;;
6242 (define_insn "mve_vshllbq_m_n_<supf><mode>"
6243   [
6244    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6245         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6246                        (match_operand:MVE_3 2 "s_register_operand" "w")
6247                        (match_operand:SI 3 "immediate_operand" "i")
6248                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6249          VSHLLBQ_M_N))
6250   ]
6251   "TARGET_HAVE_MVE"
6252   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6253   [(set_attr "type" "mve_move")
6254    (set_attr "length""8")])
6255
6256 ;;
6257 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
6258 ;;
6259 (define_insn "mve_vshlltq_m_n_<supf><mode>"
6260   [
6261    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6262         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6263                        (match_operand:MVE_3 2 "s_register_operand" "w")
6264                        (match_operand:SI 3 "immediate_operand" "i")
6265                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6266          VSHLLTQ_M_N))
6267   ]
6268   "TARGET_HAVE_MVE"
6269   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6270   [(set_attr "type" "mve_move")
6271    (set_attr "length""8")])
6272
6273 ;;
6274 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
6275 ;;
6276 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
6277   [
6278    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6279         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6280                        (match_operand:MVE_5 2 "s_register_operand" "w")
6281                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6282                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6283          VSHRNBQ_M_N))
6284   ]
6285   "TARGET_HAVE_MVE"
6286   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
6287   [(set_attr "type" "mve_move")
6288    (set_attr "length""8")])
6289
6290 ;;
6291 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
6292 ;;
6293 (define_insn "mve_vshrntq_m_n_<supf><mode>"
6294   [
6295    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6296         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6297                        (match_operand:MVE_5 2 "s_register_operand" "w")
6298                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6299                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6300          VSHRNTQ_M_N))
6301   ]
6302   "TARGET_HAVE_MVE"
6303   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
6304   [(set_attr "type" "mve_move")
6305    (set_attr "length""8")])
6306
6307 ;;
6308 ;; [vmlsldavaq_p_s])
6309 ;;
6310 (define_insn "mve_vmlsldavaq_p_s<mode>"
6311   [
6312    (set (match_operand:DI 0 "s_register_operand" "=r")
6313         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6314                        (match_operand:MVE_5 2 "s_register_operand" "w")
6315                        (match_operand:MVE_5 3 "s_register_operand" "w")
6316                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6317          VMLSLDAVAQ_P_S))
6318   ]
6319   "TARGET_HAVE_MVE"
6320   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6321   [(set_attr "type" "mve_move")
6322    (set_attr "length""8")])
6323
6324 ;;
6325 ;; [vmlsldavaxq_p_s])
6326 ;;
6327 (define_insn "mve_vmlsldavaxq_p_s<mode>"
6328   [
6329    (set (match_operand:DI 0 "s_register_operand" "=r")
6330         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6331                        (match_operand:MVE_5 2 "s_register_operand" "w")
6332                        (match_operand:MVE_5 3 "s_register_operand" "w")
6333                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6334          VMLSLDAVAXQ_P_S))
6335   ]
6336   "TARGET_HAVE_MVE"
6337   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
6338   [(set_attr "type" "mve_move")
6339    (set_attr "length""8")])
6340
6341 ;;
6342 ;; [vmullbq_poly_m_p])
6343 ;;
6344 (define_insn "mve_vmullbq_poly_m_p<mode>"
6345   [
6346    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6347         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6348                        (match_operand:MVE_3 2 "s_register_operand" "w")
6349                        (match_operand:MVE_3 3 "s_register_operand" "w")
6350                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6351          VMULLBQ_POLY_M_P))
6352   ]
6353   "TARGET_HAVE_MVE"
6354   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6355   [(set_attr "type" "mve_move")
6356    (set_attr "length""8")])
6357
6358 ;;
6359 ;; [vmulltq_poly_m_p])
6360 ;;
6361 (define_insn "mve_vmulltq_poly_m_p<mode>"
6362   [
6363    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
6364         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6365                        (match_operand:MVE_3 2 "s_register_operand" "w")
6366                        (match_operand:MVE_3 3 "s_register_operand" "w")
6367                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6368          VMULLTQ_POLY_M_P))
6369   ]
6370   "TARGET_HAVE_MVE"
6371   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
6372   [(set_attr "type" "mve_move")
6373    (set_attr "length""8")])
6374
6375 ;;
6376 ;; [vqdmullbq_m_n_s])
6377 ;;
6378 (define_insn "mve_vqdmullbq_m_n_s<mode>"
6379   [
6380    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6381         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6382                        (match_operand:MVE_5 2 "s_register_operand" "w")
6383                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6384                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6385          VQDMULLBQ_M_N_S))
6386   ]
6387   "TARGET_HAVE_MVE"
6388   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6389   [(set_attr "type" "mve_move")
6390    (set_attr "length""8")])
6391
6392 ;;
6393 ;; [vqdmullbq_m_s])
6394 ;;
6395 (define_insn "mve_vqdmullbq_m_s<mode>"
6396   [
6397    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6398         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6399                        (match_operand:MVE_5 2 "s_register_operand" "w")
6400                        (match_operand:MVE_5 3 "s_register_operand" "w")
6401                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6402          VQDMULLBQ_M_S))
6403   ]
6404   "TARGET_HAVE_MVE"
6405   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6406   [(set_attr "type" "mve_move")
6407    (set_attr "length""8")])
6408
6409 ;;
6410 ;; [vqdmulltq_m_n_s])
6411 ;;
6412 (define_insn "mve_vqdmulltq_m_n_s<mode>"
6413   [
6414    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6415         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6416                        (match_operand:MVE_5 2 "s_register_operand" "w")
6417                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6418                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6419          VQDMULLTQ_M_N_S))
6420   ]
6421   "TARGET_HAVE_MVE"
6422   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
6423   [(set_attr "type" "mve_move")
6424    (set_attr "length""8")])
6425
6426 ;;
6427 ;; [vqdmulltq_m_s])
6428 ;;
6429 (define_insn "mve_vqdmulltq_m_s<mode>"
6430   [
6431    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6432         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6433                        (match_operand:MVE_5 2 "s_register_operand" "w")
6434                        (match_operand:MVE_5 3 "s_register_operand" "w")
6435                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6436          VQDMULLTQ_M_S))
6437   ]
6438   "TARGET_HAVE_MVE"
6439   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6440   [(set_attr "type" "mve_move")
6441    (set_attr "length""8")])
6442
6443 ;;
6444 ;; [vqrshrunbq_m_n_s])
6445 ;;
6446 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
6447   [
6448    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6449         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6450                        (match_operand:MVE_5 2 "s_register_operand" "w")
6451                        (match_operand:SI 3 "mve_imm_8" "Rb")
6452                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6453          VQRSHRUNBQ_M_N_S))
6454   ]
6455   "TARGET_HAVE_MVE"
6456   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6457   [(set_attr "type" "mve_move")
6458    (set_attr "length""8")])
6459
6460 ;;
6461 ;; [vqrshruntq_m_n_s])
6462 ;;
6463 (define_insn "mve_vqrshruntq_m_n_s<mode>"
6464   [
6465    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6466         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6467                        (match_operand:MVE_5 2 "s_register_operand" "w")
6468                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6469                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6470          VQRSHRUNTQ_M_N_S))
6471   ]
6472   "TARGET_HAVE_MVE"
6473   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6474   [(set_attr "type" "mve_move")
6475    (set_attr "length""8")])
6476
6477 ;;
6478 ;; [vqshrunbq_m_n_s])
6479 ;;
6480 (define_insn "mve_vqshrunbq_m_n_s<mode>"
6481   [
6482    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6483         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6484                        (match_operand:MVE_5 2 "s_register_operand" "w")
6485                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6486                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6487          VQSHRUNBQ_M_N_S))
6488   ]
6489   "TARGET_HAVE_MVE"
6490   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
6491   [(set_attr "type" "mve_move")
6492    (set_attr "length""8")])
6493
6494 ;;
6495 ;; [vqshruntq_m_n_s])
6496 ;;
6497 (define_insn "mve_vqshruntq_m_n_s<mode>"
6498   [
6499    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
6500         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
6501                        (match_operand:MVE_5 2 "s_register_operand" "w")
6502                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
6503                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6504          VQSHRUNTQ_M_N_S))
6505   ]
6506   "TARGET_HAVE_MVE"
6507   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
6508   [(set_attr "type" "mve_move")
6509    (set_attr "length""8")])
6510
6511 ;;
6512 ;; [vrmlaldavhaq_p_u])
6513 ;;
6514 (define_insn "mve_vrmlaldavhaq_p_uv4si"
6515   [
6516    (set (match_operand:DI 0 "s_register_operand" "=r")
6517         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6518                        (match_operand:V4SI 2 "s_register_operand" "w")
6519                        (match_operand:V4SI 3 "s_register_operand" "w")
6520                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6521          VRMLALDAVHAQ_P_U))
6522   ]
6523   "TARGET_HAVE_MVE"
6524   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
6525   [(set_attr "type" "mve_move")
6526    (set_attr "length""8")])
6527
6528 ;;
6529 ;; [vrmlaldavhaxq_p_s])
6530 ;;
6531 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
6532   [
6533    (set (match_operand:DI 0 "s_register_operand" "=r")
6534         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6535                        (match_operand:V4SI 2 "s_register_operand" "w")
6536                        (match_operand:V4SI 3 "s_register_operand" "w")
6537                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6538          VRMLALDAVHAXQ_P_S))
6539   ]
6540   "TARGET_HAVE_MVE"
6541   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6542   [(set_attr "type" "mve_move")
6543    (set_attr "length""8")])
6544
6545 ;;
6546 ;; [vrmlsldavhaq_p_s])
6547 ;;
6548 (define_insn "mve_vrmlsldavhaq_p_sv4si"
6549   [
6550    (set (match_operand:DI 0 "s_register_operand" "=r")
6551         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6552                        (match_operand:V4SI 2 "s_register_operand" "w")
6553                        (match_operand:V4SI 3 "s_register_operand" "w")
6554                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6555          VRMLSLDAVHAQ_P_S))
6556   ]
6557   "TARGET_HAVE_MVE"
6558   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
6559   [(set_attr "type" "mve_move")
6560    (set_attr "length""8")])
6561
6562 ;;
6563 ;; [vrmlsldavhaxq_p_s])
6564 ;;
6565 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
6566   [
6567    (set (match_operand:DI 0 "s_register_operand" "=r")
6568         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
6569                        (match_operand:V4SI 2 "s_register_operand" "w")
6570                        (match_operand:V4SI 3 "s_register_operand" "w")
6571                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6572          VRMLSLDAVHAXQ_P_S))
6573   ]
6574   "TARGET_HAVE_MVE"
6575   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
6576   [(set_attr "type" "mve_move")
6577    (set_attr "length""8")])
6578 ;;
6579 ;; [vabdq_m_f])
6580 ;;
6581 (define_insn "mve_vabdq_m_f<mode>"
6582   [
6583    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6584         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6585                        (match_operand:MVE_0 2 "s_register_operand" "w")
6586                        (match_operand:MVE_0 3 "s_register_operand" "w")
6587                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6588          VABDQ_M_F))
6589   ]
6590   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6591   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
6592   [(set_attr "type" "mve_move")
6593    (set_attr "length""8")])
6594
6595 ;;
6596 ;; [vaddq_m_f])
6597 ;;
6598 (define_insn "mve_vaddq_m_f<mode>"
6599   [
6600    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6601         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6602                        (match_operand:MVE_0 2 "s_register_operand" "w")
6603                        (match_operand:MVE_0 3 "s_register_operand" "w")
6604                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6605          VADDQ_M_F))
6606   ]
6607   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6608   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
6609   [(set_attr "type" "mve_move")
6610    (set_attr "length""8")])
6611
6612 ;;
6613 ;; [vaddq_m_n_f])
6614 ;;
6615 (define_insn "mve_vaddq_m_n_f<mode>"
6616   [
6617    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6618         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6619                        (match_operand:MVE_0 2 "s_register_operand" "w")
6620                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6621                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6622          VADDQ_M_N_F))
6623   ]
6624   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6625   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
6626   [(set_attr "type" "mve_move")
6627    (set_attr "length""8")])
6628
6629 ;;
6630 ;; [vandq_m_f])
6631 ;;
6632 (define_insn "mve_vandq_m_f<mode>"
6633   [
6634    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6635         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6636                        (match_operand:MVE_0 2 "s_register_operand" "w")
6637                        (match_operand:MVE_0 3 "s_register_operand" "w")
6638                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6639          VANDQ_M_F))
6640   ]
6641   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6642   "vpst\;vandt %q0, %q2, %q3"
6643   [(set_attr "type" "mve_move")
6644    (set_attr "length""8")])
6645
6646 ;;
6647 ;; [vbicq_m_f])
6648 ;;
6649 (define_insn "mve_vbicq_m_f<mode>"
6650   [
6651    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6652         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6653                        (match_operand:MVE_0 2 "s_register_operand" "w")
6654                        (match_operand:MVE_0 3 "s_register_operand" "w")
6655                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6656          VBICQ_M_F))
6657   ]
6658   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6659   "vpst\;vbict %q0, %q2, %q3"
6660   [(set_attr "type" "mve_move")
6661    (set_attr "length""8")])
6662
6663 ;;
6664 ;; [vbrsrq_m_n_f])
6665 ;;
6666 (define_insn "mve_vbrsrq_m_n_f<mode>"
6667   [
6668    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6669         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6670                        (match_operand:MVE_0 2 "s_register_operand" "w")
6671                        (match_operand:SI 3 "s_register_operand" "r")
6672                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6673          VBRSRQ_M_N_F))
6674   ]
6675   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6676   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6677   [(set_attr "type" "mve_move")
6678    (set_attr "length""8")])
6679
6680 ;;
6681 ;; [vcaddq_rot270_m_f])
6682 ;;
6683 (define_insn "mve_vcaddq_rot270_m_f<mode>"
6684   [
6685    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6686         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6687                        (match_operand:MVE_0 2 "s_register_operand" "w")
6688                        (match_operand:MVE_0 3 "s_register_operand" "w")
6689                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6690          VCADDQ_ROT270_M_F))
6691   ]
6692   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6693   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6694   [(set_attr "type" "mve_move")
6695    (set_attr "length""8")])
6696
6697 ;;
6698 ;; [vcaddq_rot90_m_f])
6699 ;;
6700 (define_insn "mve_vcaddq_rot90_m_f<mode>"
6701   [
6702    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6703         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6704                        (match_operand:MVE_0 2 "s_register_operand" "w")
6705                        (match_operand:MVE_0 3 "s_register_operand" "w")
6706                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6707          VCADDQ_ROT90_M_F))
6708   ]
6709   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6710   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6711   [(set_attr "type" "mve_move")
6712    (set_attr "length""8")])
6713
6714 ;;
6715 ;; [vcmlaq_m_f])
6716 ;;
6717 (define_insn "mve_vcmlaq_m_f<mode>"
6718   [
6719    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6720         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6721                        (match_operand:MVE_0 2 "s_register_operand" "w")
6722                        (match_operand:MVE_0 3 "s_register_operand" "w")
6723                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6724          VCMLAQ_M_F))
6725   ]
6726   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6727   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6728   [(set_attr "type" "mve_move")
6729    (set_attr "length""8")])
6730
6731 ;;
6732 ;; [vcmlaq_rot180_m_f])
6733 ;;
6734 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
6735   [
6736    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6737         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6738                        (match_operand:MVE_0 2 "s_register_operand" "w")
6739                        (match_operand:MVE_0 3 "s_register_operand" "w")
6740                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6741          VCMLAQ_ROT180_M_F))
6742   ]
6743   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6744   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6745   [(set_attr "type" "mve_move")
6746    (set_attr "length""8")])
6747
6748 ;;
6749 ;; [vcmlaq_rot270_m_f])
6750 ;;
6751 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
6752   [
6753    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6754         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6755                        (match_operand:MVE_0 2 "s_register_operand" "w")
6756                        (match_operand:MVE_0 3 "s_register_operand" "w")
6757                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6758          VCMLAQ_ROT270_M_F))
6759   ]
6760   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6761   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6762   [(set_attr "type" "mve_move")
6763    (set_attr "length""8")])
6764
6765 ;;
6766 ;; [vcmlaq_rot90_m_f])
6767 ;;
6768 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
6769   [
6770    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6771         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6772                        (match_operand:MVE_0 2 "s_register_operand" "w")
6773                        (match_operand:MVE_0 3 "s_register_operand" "w")
6774                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6775          VCMLAQ_ROT90_M_F))
6776   ]
6777   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6778   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6779   [(set_attr "type" "mve_move")
6780    (set_attr "length""8")])
6781
6782 ;;
6783 ;; [vcmulq_m_f])
6784 ;;
6785 (define_insn "mve_vcmulq_m_f<mode>"
6786   [
6787    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6788         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6789                        (match_operand:MVE_0 2 "s_register_operand" "w")
6790                        (match_operand:MVE_0 3 "s_register_operand" "w")
6791                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6792          VCMULQ_M_F))
6793   ]
6794   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6795   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
6796   [(set_attr "type" "mve_move")
6797    (set_attr "length""8")])
6798
6799 ;;
6800 ;; [vcmulq_rot180_m_f])
6801 ;;
6802 (define_insn "mve_vcmulq_rot180_m_f<mode>"
6803   [
6804    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6805         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6806                        (match_operand:MVE_0 2 "s_register_operand" "w")
6807                        (match_operand:MVE_0 3 "s_register_operand" "w")
6808                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6809          VCMULQ_ROT180_M_F))
6810   ]
6811   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6812   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
6813   [(set_attr "type" "mve_move")
6814    (set_attr "length""8")])
6815
6816 ;;
6817 ;; [vcmulq_rot270_m_f])
6818 ;;
6819 (define_insn "mve_vcmulq_rot270_m_f<mode>"
6820   [
6821    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6822         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6823                        (match_operand:MVE_0 2 "s_register_operand" "w")
6824                        (match_operand:MVE_0 3 "s_register_operand" "w")
6825                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6826          VCMULQ_ROT270_M_F))
6827   ]
6828   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6829   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
6830   [(set_attr "type" "mve_move")
6831    (set_attr "length""8")])
6832
6833 ;;
6834 ;; [vcmulq_rot90_m_f])
6835 ;;
6836 (define_insn "mve_vcmulq_rot90_m_f<mode>"
6837   [
6838    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
6839         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6840                        (match_operand:MVE_0 2 "s_register_operand" "w")
6841                        (match_operand:MVE_0 3 "s_register_operand" "w")
6842                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6843          VCMULQ_ROT90_M_F))
6844   ]
6845   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6846   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
6847   [(set_attr "type" "mve_move")
6848    (set_attr "length""8")])
6849
6850 ;;
6851 ;; [veorq_m_f])
6852 ;;
6853 (define_insn "mve_veorq_m_f<mode>"
6854   [
6855    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6856         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6857                        (match_operand:MVE_0 2 "s_register_operand" "w")
6858                        (match_operand:MVE_0 3 "s_register_operand" "w")
6859                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6860          VEORQ_M_F))
6861   ]
6862   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6863   "vpst\;veort %q0, %q2, %q3"
6864   [(set_attr "type" "mve_move")
6865    (set_attr "length""8")])
6866
6867 ;;
6868 ;; [vfmaq_m_f])
6869 ;;
6870 (define_insn "mve_vfmaq_m_f<mode>"
6871   [
6872    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6873         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6874                        (match_operand:MVE_0 2 "s_register_operand" "w")
6875                        (match_operand:MVE_0 3 "s_register_operand" "w")
6876                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6877          VFMAQ_M_F))
6878   ]
6879   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6880   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
6881   [(set_attr "type" "mve_move")
6882    (set_attr "length""8")])
6883
6884 ;;
6885 ;; [vfmaq_m_n_f])
6886 ;;
6887 (define_insn "mve_vfmaq_m_n_f<mode>"
6888   [
6889    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6890         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6891                        (match_operand:MVE_0 2 "s_register_operand" "w")
6892                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6893                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6894          VFMAQ_M_N_F))
6895   ]
6896   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6897   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
6898   [(set_attr "type" "mve_move")
6899    (set_attr "length""8")])
6900
6901 ;;
6902 ;; [vfmasq_m_n_f])
6903 ;;
6904 (define_insn "mve_vfmasq_m_n_f<mode>"
6905   [
6906    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6907         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6908                        (match_operand:MVE_0 2 "s_register_operand" "w")
6909                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6910                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6911          VFMASQ_M_N_F))
6912   ]
6913   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6914   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
6915   [(set_attr "type" "mve_move")
6916    (set_attr "length""8")])
6917
6918 ;;
6919 ;; [vfmsq_m_f])
6920 ;;
6921 (define_insn "mve_vfmsq_m_f<mode>"
6922   [
6923    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6924         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6925                        (match_operand:MVE_0 2 "s_register_operand" "w")
6926                        (match_operand:MVE_0 3 "s_register_operand" "w")
6927                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6928          VFMSQ_M_F))
6929   ]
6930   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6931   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
6932   [(set_attr "type" "mve_move")
6933    (set_attr "length""8")])
6934
6935 ;;
6936 ;; [vmaxnmq_m_f])
6937 ;;
6938 (define_insn "mve_vmaxnmq_m_f<mode>"
6939   [
6940    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6941         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6942                        (match_operand:MVE_0 2 "s_register_operand" "w")
6943                        (match_operand:MVE_0 3 "s_register_operand" "w")
6944                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6945          VMAXNMQ_M_F))
6946   ]
6947   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6948   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6949   [(set_attr "type" "mve_move")
6950    (set_attr "length""8")])
6951
6952 ;;
6953 ;; [vminnmq_m_f])
6954 ;;
6955 (define_insn "mve_vminnmq_m_f<mode>"
6956   [
6957    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6958         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6959                        (match_operand:MVE_0 2 "s_register_operand" "w")
6960                        (match_operand:MVE_0 3 "s_register_operand" "w")
6961                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6962          VMINNMQ_M_F))
6963   ]
6964   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6965   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
6966   [(set_attr "type" "mve_move")
6967    (set_attr "length""8")])
6968
6969 ;;
6970 ;; [vmulq_m_f])
6971 ;;
6972 (define_insn "mve_vmulq_m_f<mode>"
6973   [
6974    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6975         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6976                        (match_operand:MVE_0 2 "s_register_operand" "w")
6977                        (match_operand:MVE_0 3 "s_register_operand" "w")
6978                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6979          VMULQ_M_F))
6980   ]
6981   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6982   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
6983   [(set_attr "type" "mve_move")
6984    (set_attr "length""8")])
6985
6986 ;;
6987 ;; [vmulq_m_n_f])
6988 ;;
6989 (define_insn "mve_vmulq_m_n_f<mode>"
6990   [
6991    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6992         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6993                        (match_operand:MVE_0 2 "s_register_operand" "w")
6994                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6995                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6996          VMULQ_M_N_F))
6997   ]
6998   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6999   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7000   [(set_attr "type" "mve_move")
7001    (set_attr "length""8")])
7002
7003 ;;
7004 ;; [vornq_m_f])
7005 ;;
7006 (define_insn "mve_vornq_m_f<mode>"
7007   [
7008    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7009         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7010                        (match_operand:MVE_0 2 "s_register_operand" "w")
7011                        (match_operand:MVE_0 3 "s_register_operand" "w")
7012                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7013          VORNQ_M_F))
7014   ]
7015   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7016   "vpst\;vornt %q0, %q2, %q3"
7017   [(set_attr "type" "mve_move")
7018    (set_attr "length""8")])
7019
7020 ;;
7021 ;; [vorrq_m_f])
7022 ;;
7023 (define_insn "mve_vorrq_m_f<mode>"
7024   [
7025    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7026         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7027                        (match_operand:MVE_0 2 "s_register_operand" "w")
7028                        (match_operand:MVE_0 3 "s_register_operand" "w")
7029                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7030          VORRQ_M_F))
7031   ]
7032   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7033   "vpst\;vorrt %q0, %q2, %q3"
7034   [(set_attr "type" "mve_move")
7035    (set_attr "length""8")])
7036
7037 ;;
7038 ;; [vsubq_m_f])
7039 ;;
7040 (define_insn "mve_vsubq_m_f<mode>"
7041   [
7042    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7043         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7044                        (match_operand:MVE_0 2 "s_register_operand" "w")
7045                        (match_operand:MVE_0 3 "s_register_operand" "w")
7046                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7047          VSUBQ_M_F))
7048   ]
7049   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7050   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
7051   [(set_attr "type" "mve_move")
7052    (set_attr "length""8")])
7053
7054 ;;
7055 ;; [vsubq_m_n_f])
7056 ;;
7057 (define_insn "mve_vsubq_m_n_f<mode>"
7058   [
7059    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7060         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7061                        (match_operand:MVE_0 2 "s_register_operand" "w")
7062                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7063                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7064          VSUBQ_M_N_F))
7065   ]
7066   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7067   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
7068   [(set_attr "type" "mve_move")
7069    (set_attr "length""8")])
7070
7071 ;;
7072 ;; [vstrbq_s vstrbq_u]
7073 ;;
7074 (define_insn "mve_vstrbq_<supf><mode>"
7075   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7076         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
7077          VSTRBQ))
7078   ]
7079   "TARGET_HAVE_MVE"
7080 {
7081    rtx ops[2];
7082    int regno = REGNO (operands[1]);
7083    ops[1] = gen_rtx_REG (TImode, regno);
7084    ops[0]  = operands[0];
7085    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
7086    return "";
7087 }
7088   [(set_attr "length" "4")])
7089
7090 ;;
7091 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
7092 ;;
7093 (define_expand "mve_vstrbq_scatter_offset_<supf><mode>"
7094   [(match_operand:<MVE_B_ELEM> 0 "mve_scatter_memory")
7095    (match_operand:MVE_2 1 "s_register_operand")
7096    (match_operand:MVE_2 2 "s_register_operand")
7097    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7098   "TARGET_HAVE_MVE"
7099 {
7100   rtx ind = XEXP (operands[0], 0);
7101   gcc_assert (REG_P (ind));
7102   emit_insn (gen_mve_vstrbq_scatter_offset_<supf><mode>_insn (ind, operands[1],
7103                                                               operands[2]));
7104   DONE;
7105 })
7106
7107 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>_insn"
7108   [(set (mem:BLK (scratch))
7109         (unspec:BLK
7110           [(match_operand:SI 0 "register_operand" "r")
7111            (match_operand:MVE_2 1 "s_register_operand" "w")
7112            (match_operand:MVE_2 2 "s_register_operand" "w")]
7113           VSTRBSOQ))]
7114   "TARGET_HAVE_MVE"
7115   "vstrb.<V_sz_elem>\t%q2, [%0, %q1]"
7116   [(set_attr "length" "4")])
7117
7118 ;;
7119 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
7120 ;;
7121 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
7122   [(set (mem:BLK (scratch))
7123         (unspec:BLK
7124                 [(match_operand:V4SI 0 "s_register_operand" "w")
7125                  (match_operand:SI 1 "immediate_operand" "i")
7126                  (match_operand:V4SI 2 "s_register_operand" "w")]
7127          VSTRWSBQ))
7128   ]
7129   "TARGET_HAVE_MVE"
7130 {
7131    rtx ops[3];
7132    ops[0] = operands[0];
7133    ops[1] = operands[1];
7134    ops[2] = operands[2];
7135    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
7136    return "";
7137 }
7138   [(set_attr "length" "4")])
7139
7140 ;;
7141 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
7142 ;;
7143 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
7144   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7145         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7146                        (match_operand:MVE_2 2 "s_register_operand" "w")]
7147          VLDRBGOQ))
7148   ]
7149   "TARGET_HAVE_MVE"
7150 {
7151    rtx ops[3];
7152    ops[0] = operands[0];
7153    ops[1] = operands[1];
7154    ops[2] = operands[2];
7155    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7156      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
7157    else
7158      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7159    return "";
7160 }
7161   [(set_attr "length" "4")])
7162
7163 ;;
7164 ;; [vldrbq_s vldrbq_u]
7165 ;;
7166 (define_insn "mve_vldrbq_<supf><mode>"
7167   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7168         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")]
7169          VLDRBQ))
7170   ]
7171   "TARGET_HAVE_MVE"
7172 {
7173    rtx ops[2];
7174    int regno = REGNO (operands[0]);
7175    ops[0] = gen_rtx_REG (TImode, regno);
7176    ops[1]  = operands[1];
7177    if (<V_sz_elem> == 8)
7178      output_asm_insn ("vldrb.<V_sz_elem>\t%q0, %E1",ops);
7179    else
7180      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
7181    return "";
7182 }
7183   [(set_attr "length" "4")])
7184
7185 ;;
7186 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
7187 ;;
7188 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
7189   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7190         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7191                       (match_operand:SI 2 "immediate_operand" "i")]
7192          VLDRWGBQ))
7193   ]
7194   "TARGET_HAVE_MVE"
7195 {
7196    rtx ops[3];
7197    ops[0] = operands[0];
7198    ops[1] = operands[1];
7199    ops[2] = operands[2];
7200    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7201    return "";
7202 }
7203   [(set_attr "length" "4")])
7204
7205 ;;
7206 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
7207 ;;
7208 (define_expand "mve_vstrbq_scatter_offset_p_<supf><mode>"
7209   [(match_operand:<MVE_B_ELEM>  0 "mve_scatter_memory")
7210    (match_operand:MVE_2 1 "s_register_operand")
7211    (match_operand:MVE_2 2 "s_register_operand")
7212    (match_operand:HI 3 "vpr_register_operand" "Up")
7213    (unspec:V4SI [(const_int 0)] VSTRBSOQ)]
7214   "TARGET_HAVE_MVE"
7215 {
7216   rtx ind = XEXP (operands[0], 0);
7217   gcc_assert (REG_P (ind));
7218   emit_insn (
7219     gen_mve_vstrbq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
7220                                                        operands[2],
7221                                                        operands[3]));
7222   DONE;
7223 })
7224
7225 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>_insn"
7226   [(set (mem:BLK (scratch))
7227         (unspec:BLK
7228           [(match_operand:SI 0 "register_operand" "r")
7229            (match_operand:MVE_2 1 "s_register_operand" "w")
7230            (match_operand:MVE_2 2 "s_register_operand" "w")
7231            (match_operand:HI 3 "vpr_register_operand" "Up")]
7232           VSTRBSOQ))]
7233   "TARGET_HAVE_MVE"
7234   "vpst\;vstrbt.<V_sz_elem>\t%q2, [%0, %q1]"
7235   [(set_attr "length" "8")])
7236
7237 ;;
7238 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
7239 ;;
7240 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
7241   [(set (mem:BLK (scratch))
7242         (unspec:BLK
7243                 [(match_operand:V4SI 0 "s_register_operand" "w")
7244                  (match_operand:SI 1 "immediate_operand" "i")
7245                  (match_operand:V4SI 2 "s_register_operand" "w")
7246                  (match_operand:HI 3 "vpr_register_operand" "Up")]
7247          VSTRWSBQ))
7248   ]
7249   "TARGET_HAVE_MVE"
7250 {
7251    rtx ops[3];
7252    ops[0] = operands[0];
7253    ops[1] = operands[1];
7254    ops[2] = operands[2];
7255    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
7256    return "";
7257 }
7258   [(set_attr "length" "8")])
7259
7260 ;;
7261 ;; [vstrbq_p_s vstrbq_p_u]
7262 ;;
7263 (define_insn "mve_vstrbq_p_<supf><mode>"
7264   [(set (match_operand:<MVE_B_ELEM> 0 "mve_memory_operand" "=Ux")
7265         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
7266                               (match_operand:HI 2 "vpr_register_operand" "Up")]
7267          VSTRBQ))
7268   ]
7269   "TARGET_HAVE_MVE"
7270 {
7271    rtx ops[2];
7272    int regno = REGNO (operands[1]);
7273    ops[1] = gen_rtx_REG (TImode, regno);
7274    ops[0]  = operands[0];
7275    output_asm_insn ("vpst\;vstrbt.<V_sz_elem>\t%q1, %E0",ops);
7276    return "";
7277 }
7278   [(set_attr "length" "8")])
7279
7280 ;;
7281 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
7282 ;;
7283 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
7284   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
7285         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
7286                        (match_operand:MVE_2 2 "s_register_operand" "w")
7287                        (match_operand:HI 3 "vpr_register_operand" "Up")]
7288          VLDRBGOQ))
7289   ]
7290   "TARGET_HAVE_MVE"
7291 {
7292    rtx ops[4];
7293    ops[0] = operands[0];
7294    ops[1] = operands[1];
7295    ops[2] = operands[2];
7296    ops[3] = operands[3];
7297    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
7298      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
7299    else
7300      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7301    return "";
7302 }
7303   [(set_attr "length" "8")])
7304
7305 ;;
7306 ;; [vldrbq_z_s vldrbq_z_u]
7307 ;;
7308 (define_insn "mve_vldrbq_z_<supf><mode>"
7309   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
7310         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "mve_memory_operand" "Ux")
7311                        (match_operand:HI 2 "vpr_register_operand" "Up")]
7312          VLDRBQ))
7313   ]
7314   "TARGET_HAVE_MVE"
7315 {
7316    rtx ops[2];
7317    int regno = REGNO (operands[0]);
7318    ops[0] = gen_rtx_REG (TImode, regno);
7319    ops[1]  = operands[1];
7320    if (<V_sz_elem> == 8)
7321      output_asm_insn ("vpst\;vldrbt.<V_sz_elem>\t%q0, %E1",ops);
7322    else
7323      output_asm_insn ("vpst\;vldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
7324    return "";
7325 }
7326   [(set_attr "length" "8")])
7327
7328 ;;
7329 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
7330 ;;
7331 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
7332   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7333         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
7334                       (match_operand:SI 2 "immediate_operand" "i")
7335                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7336          VLDRWGBQ))
7337   ]
7338   "TARGET_HAVE_MVE"
7339 {
7340    rtx ops[3];
7341    ops[0] = operands[0];
7342    ops[1] = operands[1];
7343    ops[2] = operands[2];
7344    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7345    return "";
7346 }
7347   [(set_attr "length" "8")])
7348
7349 ;;
7350 ;; [vldrhq_f]
7351 ;;
7352 (define_insn "mve_vldrhq_fv8hf"
7353   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7354         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")]
7355          VLDRHQ_F))
7356   ]
7357   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7358 {
7359    rtx ops[2];
7360    int regno = REGNO (operands[0]);
7361    ops[0] = gen_rtx_REG (TImode, regno);
7362    ops[1]  = operands[1];
7363    output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7364    return "";
7365 }
7366   [(set_attr "length" "4")])
7367
7368 ;;
7369 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
7370 ;;
7371 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
7372   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7373         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7374                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7375         VLDRHGOQ))
7376   ]
7377   "TARGET_HAVE_MVE"
7378 {
7379    rtx ops[3];
7380    ops[0] = operands[0];
7381    ops[1] = operands[1];
7382    ops[2] = operands[2];
7383    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7384      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
7385    else
7386      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7387    return "";
7388 }
7389   [(set_attr "length" "4")])
7390
7391 ;;
7392 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
7393 ;;
7394 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
7395   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7396         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7397                        (match_operand:MVE_6 2 "s_register_operand" "w")
7398                        (match_operand:HI 3 "vpr_register_operand" "Up")
7399         ]VLDRHGOQ))
7400   ]
7401   "TARGET_HAVE_MVE"
7402 {
7403    rtx ops[4];
7404    ops[0] = operands[0];
7405    ops[1] = operands[1];
7406    ops[2] = operands[2];
7407    ops[3] = operands[3];
7408    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7409      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
7410    else
7411      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
7412    return "";
7413 }
7414  [(set_attr "length" "8")])
7415
7416 ;;
7417 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
7418 ;;
7419 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
7420   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7421         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7422                        (match_operand:MVE_6 2 "s_register_operand" "w")]
7423         VLDRHGSOQ))
7424   ]
7425   "TARGET_HAVE_MVE"
7426 {
7427    rtx ops[3];
7428    ops[0] = operands[0];
7429    ops[1] = operands[1];
7430    ops[2] = operands[2];
7431       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7432      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7433    else
7434      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7435    return "";
7436 }
7437   [(set_attr "length" "4")])
7438
7439 ;;
7440 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
7441 ;;
7442 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
7443   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
7444         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
7445                        (match_operand:MVE_6 2 "s_register_operand" "w")
7446                        (match_operand:HI 3 "vpr_register_operand" "Up")
7447         ]VLDRHGSOQ))
7448   ]
7449   "TARGET_HAVE_MVE"
7450 {
7451    rtx ops[4];
7452    ops[0] = operands[0];
7453    ops[1] = operands[1];
7454    ops[2] = operands[2];
7455    ops[3] = operands[3];
7456    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
7457      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
7458    else
7459      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
7460    return "";
7461 }
7462   [(set_attr "length" "8")])
7463
7464 ;;
7465 ;; [vldrhq_s, vldrhq_u]
7466 ;;
7467 (define_insn "mve_vldrhq_<supf><mode>"
7468   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7469         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")]
7470          VLDRHQ))
7471   ]
7472   "TARGET_HAVE_MVE"
7473 {
7474    rtx ops[2];
7475    int regno = REGNO (operands[0]);
7476    ops[0] = gen_rtx_REG (TImode, regno);
7477    ops[1]  = operands[1];
7478    if (<V_sz_elem> == 16)
7479      output_asm_insn ("vldrh.16\t%q0, %E1",ops);
7480    else
7481      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
7482    return "";
7483 }
7484   [(set_attr "length" "4")])
7485
7486 ;;
7487 ;; [vldrhq_z_f]
7488 ;;
7489 (define_insn "mve_vldrhq_z_fv8hf"
7490   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
7491         (unspec:V8HF [(match_operand:V8HI 1 "mve_memory_operand" "Ux")
7492         (match_operand:HI 2 "vpr_register_operand" "Up")]
7493          VLDRHQ_F))
7494   ]
7495   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7496 {
7497    rtx ops[2];
7498    int regno = REGNO (operands[0]);
7499    ops[0] = gen_rtx_REG (TImode, regno);
7500    ops[1]  = operands[1];
7501    output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7502    return "";
7503 }
7504   [(set_attr "length" "8")])
7505
7506 ;;
7507 ;; [vldrhq_z_s vldrhq_z_u]
7508 ;;
7509 (define_insn "mve_vldrhq_z_<supf><mode>"
7510   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
7511         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "mve_memory_operand" "Ux")
7512         (match_operand:HI 2 "vpr_register_operand" "Up")]
7513          VLDRHQ))
7514   ]
7515   "TARGET_HAVE_MVE"
7516 {
7517    rtx ops[2];
7518    int regno = REGNO (operands[0]);
7519    ops[0] = gen_rtx_REG (TImode, regno);
7520    ops[1]  = operands[1];
7521    if (<V_sz_elem> == 16)
7522      output_asm_insn ("vpst\;vldrht.16\t%q0, %E1",ops);
7523    else
7524      output_asm_insn ("vpst\;vldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
7525    return "";
7526 }
7527   [(set_attr "length" "8")])
7528
7529 ;;
7530 ;; [vldrwq_f]
7531 ;;
7532 (define_insn "mve_vldrwq_fv4sf"
7533   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7534         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")]
7535          VLDRWQ_F))
7536   ]
7537   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7538 {
7539    rtx ops[2];
7540    int regno = REGNO (operands[0]);
7541    ops[0] = gen_rtx_REG (TImode, regno);
7542    ops[1]  = operands[1];
7543    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7544    return "";
7545 }
7546   [(set_attr "length" "4")])
7547
7548 ;;
7549 ;; [vldrwq_s vldrwq_u]
7550 ;;
7551 (define_insn "mve_vldrwq_<supf>v4si"
7552   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7553         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")]
7554          VLDRWQ))
7555   ]
7556   "TARGET_HAVE_MVE"
7557 {
7558    rtx ops[2];
7559    int regno = REGNO (operands[0]);
7560    ops[0] = gen_rtx_REG (TImode, regno);
7561    ops[1]  = operands[1];
7562    output_asm_insn ("vldrw.32\t%q0, %E1",ops);
7563    return "";
7564 }
7565   [(set_attr "length" "4")])
7566
7567 ;;
7568 ;; [vldrwq_z_f]
7569 ;;
7570 (define_insn "mve_vldrwq_z_fv4sf"
7571   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
7572         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Ux")
7573         (match_operand:HI 2 "vpr_register_operand" "Up")]
7574          VLDRWQ_F))
7575   ]
7576   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7577 {
7578    rtx ops[2];
7579    int regno = REGNO (operands[0]);
7580    ops[0] = gen_rtx_REG (TImode, regno);
7581    ops[1]  = operands[1];
7582    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7583    return "";
7584 }
7585   [(set_attr "length" "8")])
7586
7587 ;;
7588 ;; [vldrwq_z_s vldrwq_z_u]
7589 ;;
7590 (define_insn "mve_vldrwq_z_<supf>v4si"
7591   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
7592         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Ux")
7593         (match_operand:HI 2 "vpr_register_operand" "Up")]
7594          VLDRWQ))
7595   ]
7596   "TARGET_HAVE_MVE"
7597 {
7598    rtx ops[2];
7599    int regno = REGNO (operands[0]);
7600    ops[0] = gen_rtx_REG (TImode, regno);
7601    ops[1]  = operands[1];
7602    output_asm_insn ("vpst\;vldrwt.32\t%q0, %E1",ops);
7603    return "";
7604 }
7605   [(set_attr "length" "8")])
7606
7607 (define_expand "mve_vld1q_f<mode>"
7608   [(match_operand:MVE_0 0 "s_register_operand")
7609    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "mve_memory_operand")] VLD1Q_F)
7610   ]
7611   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
7612 {
7613   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
7614   DONE;
7615 })
7616
7617 (define_expand "mve_vld1q_<supf><mode>"
7618   [(match_operand:MVE_2 0 "s_register_operand")
7619    (unspec:MVE_2 [(match_operand:MVE_2 1 "mve_memory_operand")] VLD1Q)
7620   ]
7621   "TARGET_HAVE_MVE"
7622 {
7623   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
7624   DONE;
7625 })
7626
7627 ;;
7628 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
7629 ;;
7630 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
7631   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7632         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7633                       (match_operand:SI 2 "immediate_operand" "i")]
7634          VLDRDGBQ))
7635   ]
7636   "TARGET_HAVE_MVE"
7637 {
7638    rtx ops[3];
7639    ops[0] = operands[0];
7640    ops[1] = operands[1];
7641    ops[2] = operands[2];
7642    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
7643    return "";
7644 }
7645   [(set_attr "length" "4")])
7646
7647 ;;
7648 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
7649 ;;
7650 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
7651   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7652         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
7653                       (match_operand:SI 2 "immediate_operand" "i")
7654                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7655          VLDRDGBQ))
7656   ]
7657   "TARGET_HAVE_MVE"
7658 {
7659    rtx ops[3];
7660    ops[0] = operands[0];
7661    ops[1] = operands[1];
7662    ops[2] = operands[2];
7663    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
7664    return "";
7665 }
7666   [(set_attr "length" "8")])
7667
7668 ;;
7669 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
7670 ;;
7671 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
7672  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7673        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7674                      (match_operand:V2DI 2 "s_register_operand" "w")]
7675         VLDRDGOQ))
7676  ]
7677  "TARGET_HAVE_MVE"
7678 {
7679   rtx ops[3];
7680   ops[0] = operands[0];
7681   ops[1] = operands[1];
7682   ops[2] = operands[2];
7683   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
7684   return "";
7685 }
7686  [(set_attr "length" "4")])
7687
7688 ;;
7689 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
7690 ;;
7691 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
7692  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7693        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7694                      (match_operand:V2DI 2 "s_register_operand" "w")
7695                      (match_operand:HI 3 "vpr_register_operand" "Up")]
7696         VLDRDGOQ))
7697  ]
7698  "TARGET_HAVE_MVE"
7699 {
7700   rtx ops[3];
7701   ops[0] = operands[0];
7702   ops[1] = operands[1];
7703   ops[2] = operands[2];
7704   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
7705   return "";
7706 }
7707  [(set_attr "length" "8")])
7708
7709 ;;
7710 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
7711 ;;
7712 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
7713   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7714         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7715                       (match_operand:V2DI 2 "s_register_operand" "w")]
7716          VLDRDGSOQ))
7717   ]
7718   "TARGET_HAVE_MVE"
7719 {
7720    rtx ops[3];
7721    ops[0] = operands[0];
7722    ops[1] = operands[1];
7723    ops[2] = operands[2];
7724    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7725    return "";
7726 }
7727   [(set_attr "length" "4")])
7728
7729 ;;
7730 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
7731 ;;
7732 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
7733   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
7734         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
7735                       (match_operand:V2DI 2 "s_register_operand" "w")
7736                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7737          VLDRDGSOQ))
7738   ]
7739   "TARGET_HAVE_MVE"
7740 {
7741    rtx ops[3];
7742    ops[0] = operands[0];
7743    ops[1] = operands[1];
7744    ops[2] = operands[2];
7745    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
7746    return "";
7747 }
7748   [(set_attr "length" "8")])
7749
7750 ;;
7751 ;; [vldrhq_gather_offset_f]
7752 ;;
7753 (define_insn "mve_vldrhq_gather_offset_fv8hf"
7754   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7755         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7756                       (match_operand:V8HI 2 "s_register_operand" "w")]
7757          VLDRHQGO_F))
7758   ]
7759   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7760 {
7761    rtx ops[3];
7762    ops[0] = operands[0];
7763    ops[1] = operands[1];
7764    ops[2] = operands[2];
7765    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
7766    return "";
7767 }
7768   [(set_attr "length" "4")])
7769
7770 ;;
7771 ;; [vldrhq_gather_offset_z_f]
7772 ;;
7773 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
7774   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7775         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7776                       (match_operand:V8HI 2 "s_register_operand" "w")
7777                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7778          VLDRHQGO_F))
7779   ]
7780   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7781 {
7782    rtx ops[4];
7783    ops[0] = operands[0];
7784    ops[1] = operands[1];
7785    ops[2] = operands[2];
7786    ops[3] = operands[3];
7787    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
7788    return "";
7789 }
7790   [(set_attr "length" "8")])
7791
7792 ;;
7793 ;; [vldrhq_gather_shifted_offset_f]
7794 ;;
7795 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
7796   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7797         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7798                       (match_operand:V8HI 2 "s_register_operand" "w")]
7799          VLDRHQGSO_F))
7800   ]
7801   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7802 {
7803    rtx ops[3];
7804    ops[0] = operands[0];
7805    ops[1] = operands[1];
7806    ops[2] = operands[2];
7807    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7808    return "";
7809 }
7810   [(set_attr "length" "4")])
7811
7812 ;;
7813 ;; [vldrhq_gather_shifted_offset_z_f]
7814 ;;
7815 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
7816   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
7817         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
7818                       (match_operand:V8HI 2 "s_register_operand" "w")
7819                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7820          VLDRHQGSO_F))
7821   ]
7822   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7823 {
7824    rtx ops[4];
7825    ops[0] = operands[0];
7826    ops[1] = operands[1];
7827    ops[2] = operands[2];
7828    ops[3] = operands[3];
7829    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
7830    return "";
7831 }
7832   [(set_attr "length" "8")])
7833
7834 ;;
7835 ;; [vldrwq_gather_base_f]
7836 ;;
7837 (define_insn "mve_vldrwq_gather_base_fv4sf"
7838   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7839         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7840                       (match_operand:SI 2 "immediate_operand" "i")]
7841          VLDRWQGB_F))
7842   ]
7843   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7844 {
7845    rtx ops[3];
7846    ops[0] = operands[0];
7847    ops[1] = operands[1];
7848    ops[2] = operands[2];
7849    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
7850    return "";
7851 }
7852   [(set_attr "length" "4")])
7853
7854 ;;
7855 ;; [vldrwq_gather_base_z_f]
7856 ;;
7857 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
7858   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7859         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
7860                       (match_operand:SI 2 "immediate_operand" "i")
7861                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7862          VLDRWQGB_F))
7863   ]
7864   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7865 {
7866    rtx ops[3];
7867    ops[0] = operands[0];
7868    ops[1] = operands[1];
7869    ops[2] = operands[2];
7870    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
7871    return "";
7872 }
7873   [(set_attr "length" "8")])
7874
7875 ;;
7876 ;; [vldrwq_gather_offset_f]
7877 ;;
7878 (define_insn "mve_vldrwq_gather_offset_fv4sf"
7879   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7880         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7881                        (match_operand:V4SI 2 "s_register_operand" "w")]
7882          VLDRWQGO_F))
7883   ]
7884   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7885 {
7886    rtx ops[3];
7887    ops[0] = operands[0];
7888    ops[1] = operands[1];
7889    ops[2] = operands[2];
7890    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7891    return "";
7892 }
7893   [(set_attr "length" "4")])
7894
7895 ;;
7896 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
7897 ;;
7898 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
7899   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7900         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7901                        (match_operand:V4SI 2 "s_register_operand" "w")]
7902          VLDRWGOQ))
7903   ]
7904   "TARGET_HAVE_MVE"
7905 {
7906    rtx ops[3];
7907    ops[0] = operands[0];
7908    ops[1] = operands[1];
7909    ops[2] = operands[2];
7910    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
7911    return "";
7912 }
7913   [(set_attr "length" "4")])
7914
7915 ;;
7916 ;; [vldrwq_gather_offset_z_f]
7917 ;;
7918 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
7919   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7920         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7921                       (match_operand:V4SI 2 "s_register_operand" "w")
7922                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7923          VLDRWQGO_F))
7924   ]
7925   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7926 {
7927    rtx ops[4];
7928    ops[0] = operands[0];
7929    ops[1] = operands[1];
7930    ops[2] = operands[2];
7931    ops[3] = operands[3];
7932    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7933    return "";
7934 }
7935   [(set_attr "length" "8")])
7936
7937 ;;
7938 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
7939 ;;
7940 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
7941   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7942         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7943                       (match_operand:V4SI 2 "s_register_operand" "w")
7944                       (match_operand:HI 3 "vpr_register_operand" "Up")]
7945          VLDRWGOQ))
7946   ]
7947   "TARGET_HAVE_MVE"
7948 {
7949    rtx ops[4];
7950    ops[0] = operands[0];
7951    ops[1] = operands[1];
7952    ops[2] = operands[2];
7953    ops[3] = operands[3];
7954    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
7955    return "";
7956 }
7957   [(set_attr "length" "8")])
7958
7959 ;;
7960 ;; [vldrwq_gather_shifted_offset_f]
7961 ;;
7962 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
7963   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
7964         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
7965                       (match_operand:V4SI 2 "s_register_operand" "w")]
7966          VLDRWQGSO_F))
7967   ]
7968   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7969 {
7970    rtx ops[3];
7971    ops[0] = operands[0];
7972    ops[1] = operands[1];
7973    ops[2] = operands[2];
7974    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
7975    return "";
7976 }
7977   [(set_attr "length" "4")])
7978
7979 ;;
7980 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
7981 ;;
7982 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
7983   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
7984         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
7985                       (match_operand:V4SI 2 "s_register_operand" "w")]
7986          VLDRWGSOQ))
7987   ]
7988   "TARGET_HAVE_MVE"
7989 {
7990    rtx ops[3];
7991    ops[0] = operands[0];
7992    ops[1] = operands[1];
7993    ops[2] = operands[2];
7994    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
7995    return "";
7996 }
7997   [(set_attr "length" "4")])
7998
7999 ;;
8000 ;; [vldrwq_gather_shifted_offset_z_f]
8001 ;;
8002 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8003   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8004         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8005                       (match_operand:V4SI 2 "s_register_operand" "w")
8006                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8007          VLDRWQGSO_F))
8008   ]
8009   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8010 {
8011    rtx ops[4];
8012    ops[0] = operands[0];
8013    ops[1] = operands[1];
8014    ops[2] = operands[2];
8015    ops[3] = operands[3];
8016    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8017    return "";
8018 }
8019   [(set_attr "length" "8")])
8020
8021 ;;
8022 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8023 ;;
8024 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8025   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8026         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8027                       (match_operand:V4SI 2 "s_register_operand" "w")
8028                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8029          VLDRWGSOQ))
8030   ]
8031   "TARGET_HAVE_MVE"
8032 {
8033    rtx ops[4];
8034    ops[0] = operands[0];
8035    ops[1] = operands[1];
8036    ops[2] = operands[2];
8037    ops[3] = operands[3];
8038    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8039    return "";
8040 }
8041   [(set_attr "length" "8")])
8042
8043 ;;
8044 ;; [vstrhq_f]
8045 ;;
8046 (define_insn "mve_vstrhq_fv8hf"
8047   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8048         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8049          VSTRHQ_F))
8050   ]
8051   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8052 {
8053    rtx ops[2];
8054    int regno = REGNO (operands[1]);
8055    ops[1] = gen_rtx_REG (TImode, regno);
8056    ops[0]  = operands[0];
8057    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
8058    return "";
8059 }
8060   [(set_attr "length" "4")])
8061
8062 ;;
8063 ;; [vstrhq_p_f]
8064 ;;
8065 (define_insn "mve_vstrhq_p_fv8hf"
8066   [(set (match_operand:V8HI 0 "mve_memory_operand" "=Ux")
8067         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
8068                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8069          VSTRHQ_F))
8070   ]
8071   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8072 {
8073    rtx ops[2];
8074    int regno = REGNO (operands[1]);
8075    ops[1] = gen_rtx_REG (TImode, regno);
8076    ops[0]  = operands[0];
8077    output_asm_insn ("vpst\;vstrht.16\t%q1, %E0",ops);
8078    return "";
8079 }
8080   [(set_attr "length" "8")])
8081
8082 ;;
8083 ;; [vstrhq_p_s vstrhq_p_u]
8084 ;;
8085 (define_insn "mve_vstrhq_p_<supf><mode>"
8086   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8087         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
8088                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8089          VSTRHQ))
8090   ]
8091   "TARGET_HAVE_MVE"
8092 {
8093    rtx ops[2];
8094    int regno = REGNO (operands[1]);
8095    ops[1] = gen_rtx_REG (TImode, regno);
8096    ops[0]  = operands[0];
8097    output_asm_insn ("vpst\;vstrht.<V_sz_elem>\t%q1, %E0",ops);
8098    return "";
8099 }
8100   [(set_attr "length" "8")])
8101
8102 ;;
8103 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
8104 ;;
8105 (define_expand "mve_vstrhq_scatter_offset_p_<supf><mode>"
8106   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8107    (match_operand:MVE_6 1 "s_register_operand")
8108    (match_operand:MVE_6 2 "s_register_operand")
8109    (match_operand:HI 3 "vpr_register_operand")
8110    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8111   "TARGET_HAVE_MVE"
8112 {
8113   rtx ind = XEXP (operands[0], 0);
8114   gcc_assert (REG_P (ind));
8115   emit_insn (
8116     gen_mve_vstrhq_scatter_offset_p_<supf><mode>_insn (ind, operands[1],
8117                                                        operands[2],
8118                                                        operands[3]));
8119   DONE;
8120 })
8121
8122 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>_insn"
8123   [(set (mem:BLK (scratch))
8124         (unspec:BLK
8125           [(match_operand:SI 0 "register_operand" "r")
8126            (match_operand:MVE_6 1 "s_register_operand" "w")
8127            (match_operand:MVE_6 2 "s_register_operand" "w")
8128            (match_operand:HI 3 "vpr_register_operand" "Up")]
8129           VSTRHSOQ))]
8130   "TARGET_HAVE_MVE"
8131   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1]"
8132   [(set_attr "length" "8")])
8133
8134 ;;
8135 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
8136 ;;
8137 (define_expand "mve_vstrhq_scatter_offset_<supf><mode>"
8138   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8139    (match_operand:MVE_6 1 "s_register_operand")
8140    (match_operand:MVE_6 2 "s_register_operand")
8141    (unspec:V4SI [(const_int 0)] VSTRHSOQ)]
8142   "TARGET_HAVE_MVE"
8143 {
8144   rtx ind = XEXP (operands[0], 0);
8145   gcc_assert (REG_P (ind));
8146   emit_insn (gen_mve_vstrhq_scatter_offset_<supf><mode>_insn (ind, operands[1],
8147                                                               operands[2]));
8148   DONE;
8149 })
8150
8151 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>_insn"
8152   [(set (mem:BLK (scratch))
8153         (unspec:BLK
8154           [(match_operand:SI 0 "register_operand" "r")
8155            (match_operand:MVE_6 1 "s_register_operand" "w")
8156            (match_operand:MVE_6 2 "s_register_operand" "w")]
8157           VSTRHSOQ))]
8158   "TARGET_HAVE_MVE"
8159   "vstrh.<V_sz_elem>\t%q2, [%0, %q1]"
8160   [(set_attr "length" "4")])
8161
8162 ;;
8163 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
8164 ;;
8165 (define_expand "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
8166   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8167    (match_operand:MVE_6 1 "s_register_operand")
8168    (match_operand:MVE_6 2 "s_register_operand")
8169    (match_operand:HI 3 "vpr_register_operand")
8170    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8171   "TARGET_HAVE_MVE"
8172 {
8173   rtx ind = XEXP (operands[0], 0);
8174   gcc_assert (REG_P (ind));
8175   emit_insn (
8176     gen_mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn (ind, operands[1],
8177                                                                operands[2],
8178                                                                operands[3]));
8179   DONE;
8180 })
8181
8182 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>_insn"
8183   [(set (mem:BLK (scratch))
8184         (unspec:BLK
8185           [(match_operand:SI 0 "register_operand" "r")
8186            (match_operand:MVE_6 1 "s_register_operand" "w")
8187            (match_operand:MVE_6 2 "s_register_operand" "w")
8188            (match_operand:HI 3 "vpr_register_operand" "Up")]
8189           VSTRHSSOQ))]
8190   "TARGET_HAVE_MVE"
8191   "vpst\;vstrht.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8192   [(set_attr "length" "8")])
8193
8194 ;;
8195 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
8196 ;;
8197 (define_expand "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
8198   [(match_operand:<MVE_H_ELEM> 0 "mve_scatter_memory")
8199    (match_operand:MVE_6 1 "s_register_operand")
8200    (match_operand:MVE_6 2 "s_register_operand")
8201    (unspec:V4SI [(const_int 0)] VSTRHSSOQ)]
8202   "TARGET_HAVE_MVE"
8203 {
8204   rtx ind = XEXP (operands[0], 0);
8205   gcc_assert (REG_P (ind));
8206   emit_insn (
8207     gen_mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn (ind, operands[1],
8208                                                              operands[2]));
8209   DONE;
8210 })
8211
8212 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>_insn"
8213   [(set (mem:BLK (scratch))
8214         (unspec:BLK
8215           [(match_operand:SI 0 "register_operand" "r")
8216            (match_operand:MVE_6 1 "s_register_operand" "w")
8217            (match_operand:MVE_6 2 "s_register_operand" "w")]
8218           VSTRHSSOQ))]
8219   "TARGET_HAVE_MVE"
8220   "vstrh.<V_sz_elem>\t%q2, [%0, %q1, uxtw #1]"
8221   [(set_attr "length" "4")])
8222
8223 ;;
8224 ;; [vstrhq_s, vstrhq_u]
8225 ;;
8226 (define_insn "mve_vstrhq_<supf><mode>"
8227   [(set (match_operand:<MVE_H_ELEM> 0 "mve_memory_operand" "=Ux")
8228         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
8229          VSTRHQ))
8230   ]
8231   "TARGET_HAVE_MVE"
8232 {
8233    rtx ops[2];
8234    int regno = REGNO (operands[1]);
8235    ops[1] = gen_rtx_REG (TImode, regno);
8236    ops[0]  = operands[0];
8237    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
8238    return "";
8239 }
8240   [(set_attr "length" "4")])
8241
8242 ;;
8243 ;; [vstrwq_f]
8244 ;;
8245 (define_insn "mve_vstrwq_fv4sf"
8246   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8247         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
8248          VSTRWQ_F))
8249   ]
8250   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8251 {
8252    rtx ops[2];
8253    int regno = REGNO (operands[1]);
8254    ops[1] = gen_rtx_REG (TImode, regno);
8255    ops[0]  = operands[0];
8256    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8257    return "";
8258 }
8259   [(set_attr "length" "4")])
8260
8261 ;;
8262 ;; [vstrwq_p_f]
8263 ;;
8264 (define_insn "mve_vstrwq_p_fv4sf"
8265   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8266         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
8267                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8268          VSTRWQ_F))
8269   ]
8270   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8271 {
8272    rtx ops[2];
8273    int regno = REGNO (operands[1]);
8274    ops[1] = gen_rtx_REG (TImode, regno);
8275    ops[0]  = operands[0];
8276    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8277    return "";
8278 }
8279   [(set_attr "length" "8")])
8280
8281 ;;
8282 ;; [vstrwq_p_s vstrwq_p_u]
8283 ;;
8284 (define_insn "mve_vstrwq_p_<supf>v4si"
8285   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8286         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8287                       (match_operand:HI 2 "vpr_register_operand" "Up")]
8288          VSTRWQ))
8289   ]
8290   "TARGET_HAVE_MVE"
8291 {
8292    rtx ops[2];
8293    int regno = REGNO (operands[1]);
8294    ops[1] = gen_rtx_REG (TImode, regno);
8295    ops[0]  = operands[0];
8296    output_asm_insn ("vpst\;vstrwt.32\t%q1, %E0",ops);
8297    return "";
8298 }
8299   [(set_attr "length" "8")])
8300
8301 ;;
8302 ;; [vstrwq_s vstrwq_u]
8303 ;;
8304 (define_insn "mve_vstrwq_<supf>v4si"
8305   [(set (match_operand:V4SI 0 "memory_operand" "=Ux")
8306         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
8307          VSTRWQ))
8308   ]
8309   "TARGET_HAVE_MVE"
8310 {
8311    rtx ops[2];
8312    int regno = REGNO (operands[1]);
8313    ops[1] = gen_rtx_REG (TImode, regno);
8314    ops[0]  = operands[0];
8315    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
8316    return "";
8317 }
8318   [(set_attr "length" "4")])
8319
8320 (define_expand "mve_vst1q_f<mode>"
8321   [(match_operand:<MVE_CNVT> 0 "mve_memory_operand")
8322    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
8323   ]
8324   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8325 {
8326   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8327   DONE;
8328 })
8329
8330 (define_expand "mve_vst1q_<supf><mode>"
8331   [(match_operand:MVE_2 0 "mve_memory_operand")
8332    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
8333   ]
8334   "TARGET_HAVE_MVE"
8335 {
8336   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8337   DONE;
8338 })
8339
8340 ;;
8341 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
8342 ;;
8343 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
8344   [(set (mem:BLK (scratch))
8345         (unspec:BLK
8346                 [(match_operand:V2DI 0 "s_register_operand" "w")
8347                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8348                  (match_operand:V2DI 2 "s_register_operand" "w")
8349                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8350          VSTRDSBQ))
8351   ]
8352   "TARGET_HAVE_MVE"
8353 {
8354    rtx ops[3];
8355    ops[0] = operands[0];
8356    ops[1] = operands[1];
8357    ops[2] = operands[2];
8358    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
8359    return "";
8360 }
8361   [(set_attr "length" "8")])
8362
8363 ;;
8364 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
8365 ;;
8366 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
8367   [(set (mem:BLK (scratch))
8368         (unspec:BLK
8369                 [(match_operand:V2DI 0 "s_register_operand" "=w")
8370                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
8371                  (match_operand:V2DI 2 "s_register_operand" "w")]
8372          VSTRDSBQ))
8373   ]
8374   "TARGET_HAVE_MVE"
8375 {
8376    rtx ops[3];
8377    ops[0] = operands[0];
8378    ops[1] = operands[1];
8379    ops[2] = operands[2];
8380    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
8381    return "";
8382 }
8383   [(set_attr "length" "4")])
8384
8385 ;;
8386 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
8387 ;;
8388 (define_expand "mve_vstrdq_scatter_offset_p_<supf>v2di"
8389   [(match_operand:V2DI 0 "mve_scatter_memory")
8390    (match_operand:V2DI 1 "s_register_operand")
8391    (match_operand:V2DI 2 "s_register_operand")
8392    (match_operand:HI 3 "vpr_register_operand")
8393    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8394   "TARGET_HAVE_MVE"
8395 {
8396   rtx ind = XEXP (operands[0], 0);
8397   gcc_assert (REG_P (ind));
8398   emit_insn (gen_mve_vstrdq_scatter_offset_p_<supf>v2di_insn (ind, operands[1],
8399                                                               operands[2],
8400                                                               operands[3]));
8401   DONE;
8402 })
8403
8404 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di_insn"
8405   [(set (mem:BLK (scratch))
8406         (unspec:BLK
8407           [(match_operand:SI 0 "register_operand" "r")
8408            (match_operand:V2DI 1 "s_register_operand" "w")
8409            (match_operand:V2DI 2 "s_register_operand" "w")
8410            (match_operand:HI 3 "vpr_register_operand" "Up")]
8411           VSTRDSOQ))]
8412   "TARGET_HAVE_MVE"
8413   "vpst\;vstrdt.64\t%q2, [%0, %q1]"
8414   [(set_attr "length" "8")])
8415
8416 ;;
8417 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
8418 ;;
8419 (define_expand "mve_vstrdq_scatter_offset_<supf>v2di"
8420   [(match_operand:V2DI 0 "mve_scatter_memory")
8421    (match_operand:V2DI 1 "s_register_operand")
8422    (match_operand:V2DI 2 "s_register_operand")
8423    (unspec:V4SI [(const_int 0)] VSTRDSOQ)]
8424   "TARGET_HAVE_MVE"
8425 {
8426   rtx ind = XEXP (operands[0], 0);
8427   gcc_assert (REG_P (ind));
8428   emit_insn (gen_mve_vstrdq_scatter_offset_<supf>v2di_insn (ind, operands[1],
8429                                                             operands[2]));
8430   DONE;
8431 })
8432
8433 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di_insn"
8434   [(set (mem:BLK (scratch))
8435         (unspec:BLK
8436           [(match_operand:SI 0 "register_operand" "r")
8437            (match_operand:V2DI 1 "s_register_operand" "w")
8438            (match_operand:V2DI 2 "s_register_operand" "w")]
8439           VSTRDSOQ))]
8440   "TARGET_HAVE_MVE"
8441   "vstrd.64\t%q2, [%0, %q1]"
8442   [(set_attr "length" "4")])
8443
8444 ;;
8445 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
8446 ;;
8447 (define_expand "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
8448   [(match_operand:V2DI 0 "mve_scatter_memory")
8449    (match_operand:V2DI 1 "s_register_operand")
8450    (match_operand:V2DI 2 "s_register_operand")
8451    (match_operand:HI 3 "vpr_register_operand")
8452    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8453   "TARGET_HAVE_MVE"
8454 {
8455   rtx ind = XEXP (operands[0], 0);
8456   gcc_assert (REG_P (ind));
8457   emit_insn (
8458     gen_mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn (ind, operands[1],
8459                                                              operands[2],
8460                                                              operands[3]));
8461   DONE;
8462 })
8463
8464 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di_insn"
8465   [(set (mem:BLK (scratch))
8466         (unspec:BLK
8467           [(match_operand:SI 0 "register_operand" "r")
8468            (match_operand:V2DI 1 "s_register_operand" "w")
8469            (match_operand:V2DI 2 "s_register_operand" "w")
8470            (match_operand:HI 3 "vpr_register_operand" "Up")]
8471           VSTRDSSOQ))]
8472   "TARGET_HAVE_MVE"
8473   "vpst\;vstrdt.64\t%q2, [%0, %q1, UXTW #3]"
8474   [(set_attr "length" "8")])
8475
8476 ;;
8477 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
8478 ;;
8479 (define_expand "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
8480   [(match_operand:V2DI 0 "mve_scatter_memory")
8481    (match_operand:V2DI 1 "s_register_operand")
8482    (match_operand:V2DI 2 "s_register_operand")
8483    (unspec:V4SI [(const_int 0)] VSTRDSSOQ)]
8484   "TARGET_HAVE_MVE"
8485 {
8486   rtx ind = XEXP (operands[0], 0);
8487   gcc_assert (REG_P (ind));
8488   emit_insn (
8489     gen_mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn (ind, operands[1],
8490                                                            operands[2]));
8491   DONE;
8492 })
8493
8494 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di_insn"
8495   [(set (mem:BLK (scratch))
8496         (unspec:BLK
8497           [(match_operand:SI 0 "register_operand" "r")
8498            (match_operand:V2DI 1 "s_register_operand" "w")
8499            (match_operand:V2DI 2 "s_register_operand" "w")]
8500           VSTRDSSOQ))]
8501   "TARGET_HAVE_MVE"
8502   "vstrd.64\t%q2, [%0, %q1, UXTW #3]"
8503   [(set_attr "length" "4")])
8504
8505 ;;
8506 ;; [vstrhq_scatter_offset_f]
8507 ;;
8508 (define_expand "mve_vstrhq_scatter_offset_fv8hf"
8509   [(match_operand:V8HI 0 "mve_scatter_memory")
8510    (match_operand:V8HI 1 "s_register_operand")
8511    (match_operand:V8HF 2 "s_register_operand")
8512    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8513   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8514 {
8515   rtx ind = XEXP (operands[0], 0);
8516   gcc_assert (REG_P (ind));
8517   emit_insn (gen_mve_vstrhq_scatter_offset_fv8hf_insn (ind, operands[1],
8518                                                        operands[2]));
8519   DONE;
8520 })
8521
8522 (define_insn "mve_vstrhq_scatter_offset_fv8hf_insn"
8523   [(set (mem:BLK (scratch))
8524         (unspec:BLK
8525           [(match_operand:SI 0 "register_operand" "r")
8526            (match_operand:V8HI 1 "s_register_operand" "w")
8527            (match_operand:V8HF 2 "s_register_operand" "w")]
8528           VSTRHQSO_F))]
8529   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8530   "vstrh.16\t%q2, [%0, %q1]"
8531   [(set_attr "length" "4")])
8532
8533 ;;
8534 ;; [vstrhq_scatter_offset_p_f]
8535 ;;
8536 (define_expand "mve_vstrhq_scatter_offset_p_fv8hf"
8537   [(match_operand:V8HI 0 "mve_scatter_memory")
8538    (match_operand:V8HI 1 "s_register_operand")
8539    (match_operand:V8HF 2 "s_register_operand")
8540    (match_operand:HI 3 "vpr_register_operand")
8541    (unspec:V4SI [(const_int 0)] VSTRHQSO_F)]
8542   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8543 {
8544   rtx ind = XEXP (operands[0], 0);
8545   gcc_assert (REG_P (ind));
8546   emit_insn (gen_mve_vstrhq_scatter_offset_p_fv8hf_insn (ind, operands[1],
8547                                                          operands[2],
8548                                                          operands[3]));
8549   DONE;
8550 })
8551
8552 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf_insn"
8553   [(set (mem:BLK (scratch))
8554         (unspec:BLK
8555           [(match_operand:SI 0 "register_operand" "r")
8556            (match_operand:V8HI 1 "s_register_operand" "w")
8557            (match_operand:V8HF 2 "s_register_operand" "w")
8558            (match_operand:HI 3 "vpr_register_operand" "Up")]
8559           VSTRHQSO_F))]
8560   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8561   "vpst\;vstrht.16\t%q2, [%0, %q1]"
8562   [(set_attr "length" "8")])
8563
8564 ;;
8565 ;; [vstrhq_scatter_shifted_offset_f]
8566 ;;
8567 (define_expand "mve_vstrhq_scatter_shifted_offset_fv8hf"
8568   [(match_operand:V8HI 0 "memory_operand" "=Us")
8569    (match_operand:V8HI 1 "s_register_operand" "w")
8570    (match_operand:V8HF 2 "s_register_operand" "w")
8571    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8572   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8573 {
8574   rtx ind = XEXP (operands[0], 0);
8575   gcc_assert (REG_P (ind));
8576   emit_insn (gen_mve_vstrhq_scatter_shifted_offset_fv8hf_insn (ind, operands[1],
8577                                                                operands[2]));
8578   DONE;
8579 })
8580
8581 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf_insn"
8582   [(set (mem:BLK (scratch))
8583         (unspec:BLK
8584           [(match_operand:SI 0 "register_operand" "r")
8585            (match_operand:V8HI 1 "s_register_operand" "w")
8586            (match_operand:V8HF 2 "s_register_operand" "w")]
8587           VSTRHQSSO_F))]
8588   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8589   "vstrh.16\t%q2, [%0, %q1, uxtw #1]"
8590   [(set_attr "length" "4")])
8591
8592 ;;
8593 ;; [vstrhq_scatter_shifted_offset_p_f]
8594 ;;
8595 (define_expand "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
8596   [(match_operand:V8HI 0 "memory_operand" "=Us")
8597    (match_operand:V8HI 1 "s_register_operand" "w")
8598    (match_operand:V8HF 2 "s_register_operand" "w")
8599    (match_operand:HI 3 "vpr_register_operand" "Up")
8600    (unspec:V4SI [(const_int 0)] VSTRHQSSO_F)]
8601   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8602 {
8603   rtx ind = XEXP (operands[0], 0);
8604   gcc_assert (REG_P (ind));
8605   emit_insn (
8606     gen_mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn (ind, operands[1],
8607                                                         operands[2],
8608                                                         operands[3]));
8609   DONE;
8610 })
8611
8612 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf_insn"
8613   [(set (mem:BLK (scratch))
8614         (unspec:BLK
8615           [(match_operand:SI 0 "register_operand" "r")
8616            (match_operand:V8HI 1 "s_register_operand" "w")
8617            (match_operand:V8HF 2 "s_register_operand" "w")
8618            (match_operand:HI 3 "vpr_register_operand" "Up")]
8619           VSTRHQSSO_F))]
8620   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8621   "vpst\;vstrht.16\t%q2, [%0, %q1, uxtw #1]"
8622   [(set_attr "length" "8")])
8623
8624 ;;
8625 ;; [vstrwq_scatter_base_f]
8626 ;;
8627 (define_insn "mve_vstrwq_scatter_base_fv4sf"
8628   [(set (mem:BLK (scratch))
8629         (unspec:BLK
8630                 [(match_operand:V4SI 0 "s_register_operand" "w")
8631                  (match_operand:SI 1 "immediate_operand" "i")
8632                  (match_operand:V4SF 2 "s_register_operand" "w")]
8633          VSTRWQSB_F))
8634   ]
8635   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8636 {
8637    rtx ops[3];
8638    ops[0] = operands[0];
8639    ops[1] = operands[1];
8640    ops[2] = operands[2];
8641    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
8642    return "";
8643 }
8644   [(set_attr "length" "4")])
8645
8646 ;;
8647 ;; [vstrwq_scatter_base_p_f]
8648 ;;
8649 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
8650   [(set (mem:BLK (scratch))
8651         (unspec:BLK
8652                 [(match_operand:V4SI 0 "s_register_operand" "w")
8653                  (match_operand:SI 1 "immediate_operand" "i")
8654                  (match_operand:V4SF 2 "s_register_operand" "w")
8655                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8656          VSTRWQSB_F))
8657   ]
8658   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8659 {
8660    rtx ops[3];
8661    ops[0] = operands[0];
8662    ops[1] = operands[1];
8663    ops[2] = operands[2];
8664    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8665    return "";
8666 }
8667   [(set_attr "length" "8")])
8668
8669 ;;
8670 ;; [vstrwq_scatter_offset_f]
8671 ;;
8672 (define_expand "mve_vstrwq_scatter_offset_fv4sf"
8673   [(match_operand:V4SI 0 "mve_scatter_memory")
8674    (match_operand:V4SI 1 "s_register_operand")
8675    (match_operand:V4SF 2 "s_register_operand")
8676    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8677   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8678 {
8679   rtx ind = XEXP (operands[0], 0);
8680   gcc_assert (REG_P (ind));
8681   emit_insn (gen_mve_vstrwq_scatter_offset_fv4sf_insn (ind, operands[1],
8682                                                        operands[2]));
8683   DONE;
8684 })
8685
8686 (define_insn "mve_vstrwq_scatter_offset_fv4sf_insn"
8687   [(set (mem:BLK (scratch))
8688         (unspec:BLK
8689           [(match_operand:SI 0 "register_operand" "r")
8690            (match_operand:V4SI 1 "s_register_operand" "w")
8691            (match_operand:V4SF 2 "s_register_operand" "w")]
8692           VSTRWQSO_F))]
8693   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8694   "vstrw.32\t%q2, [%0, %q1]"
8695   [(set_attr "length" "4")])
8696
8697 ;;
8698 ;; [vstrwq_scatter_offset_p_f]
8699 ;;
8700 (define_expand "mve_vstrwq_scatter_offset_p_fv4sf"
8701   [(match_operand:V4SI 0 "mve_scatter_memory")
8702    (match_operand:V4SI 1 "s_register_operand")
8703    (match_operand:V4SF 2 "s_register_operand")
8704    (match_operand:HI 3 "vpr_register_operand")
8705    (unspec:V4SI [(const_int 0)] VSTRWQSO_F)]
8706   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8707 {
8708   rtx ind = XEXP (operands[0], 0);
8709   gcc_assert (REG_P (ind));
8710   emit_insn (gen_mve_vstrwq_scatter_offset_p_fv4sf_insn (ind, operands[1],
8711                                                          operands[2],
8712                                                          operands[3]));
8713   DONE;
8714 })
8715
8716 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf_insn"
8717   [(set (mem:BLK (scratch))
8718         (unspec:BLK
8719           [(match_operand:SI 0 "register_operand" "r")
8720            (match_operand:V4SI 1 "s_register_operand" "w")
8721            (match_operand:V4SF 2 "s_register_operand" "w")
8722            (match_operand:HI 3 "vpr_register_operand" "Up")]
8723           VSTRWQSO_F))]
8724   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8725   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8726   [(set_attr "length" "8")])
8727
8728 ;;
8729 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8730 ;;
8731 (define_expand "mve_vstrwq_scatter_offset_p_<supf>v4si"
8732   [(match_operand:V4SI 0 "mve_scatter_memory")
8733    (match_operand:V4SI 1 "s_register_operand")
8734    (match_operand:V4SI 2 "s_register_operand")
8735    (match_operand:HI 3 "vpr_register_operand")
8736    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8737   "TARGET_HAVE_MVE"
8738 {
8739   rtx ind = XEXP (operands[0], 0);
8740   gcc_assert (REG_P (ind));
8741   emit_insn (gen_mve_vstrwq_scatter_offset_p_<supf>v4si_insn (ind, operands[1],
8742                                                               operands[2],
8743                                                               operands[3]));
8744   DONE;
8745 })
8746
8747 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si_insn"
8748   [(set (mem:BLK (scratch))
8749         (unspec:BLK
8750           [(match_operand:SI 0 "register_operand" "r")
8751            (match_operand:V4SI 1 "s_register_operand" "w")
8752            (match_operand:V4SI 2 "s_register_operand" "w")
8753            (match_operand:HI 3 "vpr_register_operand" "Up")]
8754           VSTRWSOQ))]
8755   "TARGET_HAVE_MVE"
8756   "vpst\;vstrwt.32\t%q2, [%0, %q1]"
8757   [(set_attr "length" "8")])
8758
8759 ;;
8760 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
8761 ;;
8762 (define_expand "mve_vstrwq_scatter_offset_<supf>v4si"
8763   [(match_operand:V4SI 0 "mve_scatter_memory")
8764    (match_operand:V4SI 1 "s_register_operand")
8765    (match_operand:V4SI 2 "s_register_operand")
8766    (unspec:V4SI [(const_int 0)] VSTRWSOQ)]
8767   "TARGET_HAVE_MVE"
8768 {
8769   rtx ind = XEXP (operands[0], 0);
8770   gcc_assert (REG_P (ind));
8771   emit_insn (gen_mve_vstrwq_scatter_offset_<supf>v4si_insn (ind, operands[1],
8772                                                             operands[2]));
8773   DONE;
8774 })
8775
8776 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si_insn"
8777   [(set (mem:BLK (scratch))
8778         (unspec:BLK
8779           [(match_operand:SI 0 "register_operand" "r")
8780            (match_operand:V4SI 1 "s_register_operand" "w")
8781            (match_operand:V4SI 2 "s_register_operand" "w")]
8782           VSTRWSOQ))]
8783   "TARGET_HAVE_MVE"
8784   "vstrw.32\t%q2, [%0, %q1]"
8785   [(set_attr "length" "4")])
8786
8787 ;;
8788 ;; [vstrwq_scatter_shifted_offset_f]
8789 ;;
8790 (define_expand "mve_vstrwq_scatter_shifted_offset_fv4sf"
8791   [(match_operand:V4SI 0 "mve_scatter_memory")
8792    (match_operand:V4SI 1 "s_register_operand")
8793    (match_operand:V4SF 2 "s_register_operand")
8794    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8795   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8796 {
8797   rtx ind = XEXP (operands[0], 0);
8798   gcc_assert (REG_P (ind));
8799   emit_insn (gen_mve_vstrwq_scatter_shifted_offset_fv4sf_insn (ind, operands[1],
8800                                                                operands[2]));
8801   DONE;
8802 })
8803
8804 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf_insn"
8805   [(set (mem:BLK (scratch))
8806         (unspec:BLK
8807           [(match_operand:SI 0 "register_operand" "r")
8808            (match_operand:V4SI 1 "s_register_operand" "w")
8809            (match_operand:V4SF 2 "s_register_operand" "w")]
8810          VSTRWQSSO_F))]
8811   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8812   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8813   [(set_attr "length" "8")])
8814
8815 ;;
8816 ;; [vstrwq_scatter_shifted_offset_p_f]
8817 ;;
8818 (define_expand "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
8819   [(match_operand:V4SI 0 "mve_scatter_memory")
8820    (match_operand:V4SI 1 "s_register_operand")
8821    (match_operand:V4SF 2 "s_register_operand")
8822    (match_operand:HI 3 "vpr_register_operand")
8823    (unspec:V4SI [(const_int 0)] VSTRWQSSO_F)]
8824   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8825 {
8826   rtx ind = XEXP (operands[0], 0);
8827   gcc_assert (REG_P (ind));
8828   emit_insn (
8829     gen_mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn (ind, operands[1],
8830                                                         operands[2],
8831                                                         operands[3]));
8832   DONE;
8833 })
8834
8835 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf_insn"
8836   [(set (mem:BLK (scratch))
8837         (unspec:BLK
8838           [(match_operand:SI 0 "register_operand" "r")
8839            (match_operand:V4SI 1 "s_register_operand" "w")
8840            (match_operand:V4SF 2 "s_register_operand" "w")
8841            (match_operand:HI 3 "vpr_register_operand" "Up")]
8842           VSTRWQSSO_F))]
8843   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8844   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8845   [(set_attr "length" "8")])
8846
8847 ;;
8848 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
8849 ;;
8850 (define_expand "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
8851   [(match_operand:V4SI 0 "mve_scatter_memory")
8852    (match_operand:V4SI 1 "s_register_operand")
8853    (match_operand:V4SI 2 "s_register_operand")
8854    (match_operand:HI 3 "vpr_register_operand")
8855    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8856   "TARGET_HAVE_MVE"
8857 {
8858   rtx ind = XEXP (operands[0], 0);
8859   gcc_assert (REG_P (ind));
8860   emit_insn (
8861     gen_mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn (ind, operands[1],
8862                                                              operands[2],
8863                                                              operands[3]));
8864   DONE;
8865 })
8866
8867 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si_insn"
8868   [(set (mem:BLK (scratch))
8869         (unspec:BLK
8870           [(match_operand:SI 0 "register_operand" "r")
8871            (match_operand:V4SI 1 "s_register_operand" "w")
8872            (match_operand:V4SI 2 "s_register_operand" "w")
8873            (match_operand:HI 3 "vpr_register_operand" "Up")]
8874           VSTRWSSOQ))]
8875   "TARGET_HAVE_MVE"
8876   "vpst\;vstrwt.32\t%q2, [%0, %q1, uxtw #2]"
8877   [(set_attr "length" "8")])
8878
8879 ;;
8880 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
8881 ;;
8882 (define_expand "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
8883   [(match_operand:V4SI 0 "mve_scatter_memory")
8884    (match_operand:V4SI 1 "s_register_operand")
8885    (match_operand:V4SI 2 "s_register_operand")
8886    (unspec:V4SI [(const_int 0)] VSTRWSSOQ)]
8887   "TARGET_HAVE_MVE"
8888 {
8889   rtx ind = XEXP (operands[0], 0);
8890   gcc_assert (REG_P (ind));
8891   emit_insn (
8892     gen_mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn (ind, operands[1],
8893                                                            operands[2]));
8894   DONE;
8895 })
8896
8897 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si_insn"
8898   [(set (mem:BLK (scratch))
8899         (unspec:BLK
8900           [(match_operand:SI 0 "register_operand" "r")
8901            (match_operand:V4SI 1 "s_register_operand" "w")
8902            (match_operand:V4SI 2 "s_register_operand" "w")]
8903           VSTRWSSOQ))]
8904   "TARGET_HAVE_MVE"
8905   "vstrw.32\t%q2, [%0, %q1, uxtw #2]"
8906   [(set_attr "length" "4")])
8907
8908 ;;
8909 ;; [vaddq_s, vaddq_u])
8910 ;;
8911 (define_insn "mve_vaddq<mode>"
8912   [
8913    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
8914         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
8915                     (match_operand:MVE_2 2 "s_register_operand" "w")))
8916   ]
8917   "TARGET_HAVE_MVE"
8918   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
8919   [(set_attr "type" "mve_move")
8920 ])
8921
8922 ;;
8923 ;; [vaddq_f])
8924 ;;
8925 (define_insn "mve_vaddq_f<mode>"
8926   [
8927    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8928         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
8929                     (match_operand:MVE_0 2 "s_register_operand" "w")))
8930   ]
8931   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8932   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
8933   [(set_attr "type" "mve_move")
8934 ])
8935
8936 ;;
8937 ;; [vidupq_n_u])
8938 ;;
8939 (define_expand "mve_vidupq_n_u<mode>"
8940  [(match_operand:MVE_2 0 "s_register_operand")
8941   (match_operand:SI 1 "s_register_operand")
8942   (match_operand:SI 2 "mve_imm_selective_upto_8")]
8943  "TARGET_HAVE_MVE"
8944 {
8945   rtx temp = gen_reg_rtx (SImode);
8946   emit_move_insn (temp, operands[1]);
8947   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
8948   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
8949                                           operands[2], inc));
8950   DONE;
8951 })
8952
8953 ;;
8954 ;; [vidupq_u_insn])
8955 ;;
8956 (define_insn "mve_vidupq_u<mode>_insn"
8957  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8958        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
8959                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
8960          VIDUPQ))
8961   (set (match_operand:SI 1 "s_register_operand" "=Te")
8962        (plus:SI (match_dup 2)
8963                 (match_operand:SI 4 "immediate_operand" "i")))]
8964  "TARGET_HAVE_MVE"
8965  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
8966
8967 ;;
8968 ;; [vidupq_m_n_u])
8969 ;;
8970 (define_expand "mve_vidupq_m_n_u<mode>"
8971   [(match_operand:MVE_2 0 "s_register_operand")
8972    (match_operand:MVE_2 1 "s_register_operand")
8973    (match_operand:SI 2 "s_register_operand")
8974    (match_operand:SI 3 "mve_imm_selective_upto_8")
8975    (match_operand:HI 4 "vpr_register_operand")]
8976   "TARGET_HAVE_MVE"
8977 {
8978   rtx temp = gen_reg_rtx (SImode);
8979   emit_move_insn (temp, operands[2]);
8980   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
8981   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
8982                                              operands[2], operands[3],
8983                                              operands[4], inc));
8984   DONE;
8985 })
8986
8987 ;;
8988 ;; [vidupq_m_wb_u_insn])
8989 ;;
8990 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
8991  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8992        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
8993                       (match_operand:SI 3 "s_register_operand" "2")
8994                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
8995                       (match_operand:HI 5 "vpr_register_operand" "Up")]
8996         VIDUPQ_M))
8997   (set (match_operand:SI 2 "s_register_operand" "=Te")
8998        (plus:SI (match_dup 3)
8999                 (match_operand:SI 6 "immediate_operand" "i")))]
9000  "TARGET_HAVE_MVE"
9001  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9002  [(set_attr "length""8")])
9003
9004 ;;
9005 ;; [vddupq_n_u])
9006 ;;
9007 (define_expand "mve_vddupq_n_u<mode>"
9008  [(match_operand:MVE_2 0 "s_register_operand")
9009   (match_operand:SI 1 "s_register_operand")
9010   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9011  "TARGET_HAVE_MVE"
9012 {
9013   rtx temp = gen_reg_rtx (SImode);
9014   emit_move_insn (temp, operands[1]);
9015   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9016   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9017                                           operands[2], inc));
9018   DONE;
9019 })
9020
9021 ;;
9022 ;; [vddupq_u_insn])
9023 ;;
9024 (define_insn "mve_vddupq_u<mode>_insn"
9025  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9026        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9027                       (match_operand:SI 3 "immediate_operand" "i")]
9028         VDDUPQ))
9029   (set (match_operand:SI 1 "s_register_operand" "=Te")
9030        (minus:SI (match_dup 2)
9031                  (match_operand:SI 4 "immediate_operand" "i")))]
9032  "TARGET_HAVE_MVE"
9033  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9034
9035 ;;
9036 ;; [vddupq_m_n_u])
9037 ;;
9038 (define_expand "mve_vddupq_m_n_u<mode>"
9039   [(match_operand:MVE_2 0 "s_register_operand")
9040    (match_operand:MVE_2 1 "s_register_operand")
9041    (match_operand:SI 2 "s_register_operand")
9042    (match_operand:SI 3 "mve_imm_selective_upto_8")
9043    (match_operand:HI 4 "vpr_register_operand")]
9044   "TARGET_HAVE_MVE"
9045 {
9046   rtx temp = gen_reg_rtx (SImode);
9047   emit_move_insn (temp, operands[2]);
9048   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9049   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9050                                              operands[2], operands[3],
9051                                              operands[4], inc));
9052   DONE;
9053 })
9054
9055 ;;
9056 ;; [vddupq_m_wb_u_insn])
9057 ;;
9058 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9059  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9060        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9061                       (match_operand:SI 3 "s_register_operand" "2")
9062                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9063                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9064         VDDUPQ_M))
9065   (set (match_operand:SI 2 "s_register_operand" "=Te")
9066        (minus:SI (match_dup 3)
9067                  (match_operand:SI 6 "immediate_operand" "i")))]
9068  "TARGET_HAVE_MVE"
9069  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9070  [(set_attr "length""8")])
9071
9072 ;;
9073 ;; [vdwdupq_n_u])
9074 ;;
9075 (define_expand "mve_vdwdupq_n_u<mode>"
9076  [(match_operand:MVE_2 0 "s_register_operand")
9077   (match_operand:SI 1 "s_register_operand")
9078   (match_operand:DI 2 "s_register_operand")
9079   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9080  "TARGET_HAVE_MVE"
9081 {
9082   rtx ignore_wb = gen_reg_rtx (SImode);
9083   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9084                                               operands[1], operands[2],
9085                                               operands[3]));
9086   DONE;
9087 })
9088
9089 ;;
9090 ;; [vdwdupq_wb_u])
9091 ;;
9092 (define_expand "mve_vdwdupq_wb_u<mode>"
9093  [(match_operand:SI 0 "s_register_operand")
9094   (match_operand:SI 1 "s_register_operand")
9095   (match_operand:DI 2 "s_register_operand")
9096   (match_operand:SI 3 "mve_imm_selective_upto_8")
9097   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9098  "TARGET_HAVE_MVE"
9099 {
9100   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9101   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9102                                               operands[1], operands[2],
9103                                               operands[3]));
9104   DONE;
9105 })
9106
9107 ;;
9108 ;; [vdwdupq_wb_u_insn])
9109 ;;
9110 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9111   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9112         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9113                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9114                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9115          VDWDUPQ))
9116    (set (match_operand:SI 1 "s_register_operand" "=Te")
9117         (unspec:SI [(match_dup 2)
9118                     (subreg:SI (match_dup 3) 4)
9119                     (match_dup 4)]
9120          VDWDUPQ))]
9121   "TARGET_HAVE_MVE"
9122   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9123 )
9124
9125 ;;
9126 ;; [vdwdupq_m_n_u])
9127 ;;
9128 (define_expand "mve_vdwdupq_m_n_u<mode>"
9129  [(match_operand:MVE_2 0 "s_register_operand")
9130   (match_operand:MVE_2 1 "s_register_operand")
9131   (match_operand:SI 2 "s_register_operand")
9132   (match_operand:DI 3 "s_register_operand")
9133   (match_operand:SI 4 "mve_imm_selective_upto_8")
9134   (match_operand:HI 5 "vpr_register_operand")]
9135  "TARGET_HAVE_MVE"
9136 {
9137   rtx ignore_wb = gen_reg_rtx (SImode);
9138   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9139                                                 operands[1], operands[2],
9140                                                 operands[3], operands[4],
9141                                                 operands[5]));
9142   DONE;
9143 })
9144
9145 ;;
9146 ;; [vdwdupq_m_wb_u])
9147 ;;
9148 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9149  [(match_operand:SI 0 "s_register_operand")
9150   (match_operand:MVE_2 1 "s_register_operand")
9151   (match_operand:SI 2 "s_register_operand")
9152   (match_operand:DI 3 "s_register_operand")
9153   (match_operand:SI 4 "mve_imm_selective_upto_8")
9154   (match_operand:HI 5 "vpr_register_operand")]
9155  "TARGET_HAVE_MVE"
9156 {
9157   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9158   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9159                                                 operands[1], operands[2],
9160                                                 operands[3], operands[4],
9161                                                 operands[5]));
9162   DONE;
9163 })
9164
9165 ;;
9166 ;; [vdwdupq_m_wb_u_insn])
9167 ;;
9168 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9169   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9170         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9171                        (match_operand:SI 3 "s_register_operand" "1")
9172                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9173                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9174                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9175          VDWDUPQ_M))
9176    (set (match_operand:SI 1 "s_register_operand" "=Te")
9177         (unspec:SI [(match_dup 2)
9178                     (match_dup 3)
9179                     (subreg:SI (match_dup 4) 4)
9180                     (match_dup 5)
9181                     (match_dup 6)]
9182          VDWDUPQ_M))
9183   ]
9184   "TARGET_HAVE_MVE"
9185   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9186   [(set_attr "type" "mve_move")
9187    (set_attr "length""8")])
9188
9189 ;;
9190 ;; [viwdupq_n_u])
9191 ;;
9192 (define_expand "mve_viwdupq_n_u<mode>"
9193  [(match_operand:MVE_2 0 "s_register_operand")
9194   (match_operand:SI 1 "s_register_operand")
9195   (match_operand:DI 2 "s_register_operand")
9196   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9197  "TARGET_HAVE_MVE"
9198 {
9199   rtx ignore_wb = gen_reg_rtx (SImode);
9200   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9201                                               operands[1], operands[2],
9202                                               operands[3]));
9203   DONE;
9204 })
9205
9206 ;;
9207 ;; [viwdupq_wb_u])
9208 ;;
9209 (define_expand "mve_viwdupq_wb_u<mode>"
9210  [(match_operand:SI 0 "s_register_operand")
9211   (match_operand:SI 1 "s_register_operand")
9212   (match_operand:DI 2 "s_register_operand")
9213   (match_operand:SI 3 "mve_imm_selective_upto_8")
9214   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9215  "TARGET_HAVE_MVE"
9216 {
9217   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9218   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9219                                               operands[1], operands[2],
9220                                               operands[3]));
9221   DONE;
9222 })
9223
9224 ;;
9225 ;; [viwdupq_wb_u_insn])
9226 ;;
9227 (define_insn "mve_viwdupq_wb_u<mode>_insn"
9228   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9229         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9230                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9231                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9232          VIWDUPQ))
9233    (set (match_operand:SI 1 "s_register_operand" "=Te")
9234         (unspec:SI [(match_dup 2)
9235                     (subreg:SI (match_dup 3) 4)
9236                     (match_dup 4)]
9237          VIWDUPQ))]
9238   "TARGET_HAVE_MVE"
9239   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9240 )
9241
9242 ;;
9243 ;; [viwdupq_m_n_u])
9244 ;;
9245 (define_expand "mve_viwdupq_m_n_u<mode>"
9246  [(match_operand:MVE_2 0 "s_register_operand")
9247   (match_operand:MVE_2 1 "s_register_operand")
9248   (match_operand:SI 2 "s_register_operand")
9249   (match_operand:DI 3 "s_register_operand")
9250   (match_operand:SI 4 "mve_imm_selective_upto_8")
9251   (match_operand:HI 5 "vpr_register_operand")]
9252  "TARGET_HAVE_MVE"
9253 {
9254   rtx ignore_wb = gen_reg_rtx (SImode);
9255   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9256                                                 operands[1], operands[2],
9257                                                 operands[3], operands[4],
9258                                                 operands[5]));
9259   DONE;
9260 })
9261
9262 ;;
9263 ;; [viwdupq_m_wb_u])
9264 ;;
9265 (define_expand "mve_viwdupq_m_wb_u<mode>"
9266  [(match_operand:SI 0 "s_register_operand")
9267   (match_operand:MVE_2 1 "s_register_operand")
9268   (match_operand:SI 2 "s_register_operand")
9269   (match_operand:DI 3 "s_register_operand")
9270   (match_operand:SI 4 "mve_imm_selective_upto_8")
9271   (match_operand:HI 5 "vpr_register_operand")]
9272  "TARGET_HAVE_MVE"
9273 {
9274   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9275   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9276                                                 operands[1], operands[2],
9277                                                 operands[3], operands[4],
9278                                                 operands[5]));
9279   DONE;
9280 })
9281
9282 ;;
9283 ;; [viwdupq_m_wb_u_insn])
9284 ;;
9285 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
9286   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9287         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9288                        (match_operand:SI 3 "s_register_operand" "1")
9289                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9290                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9291                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9292          VIWDUPQ_M))
9293    (set (match_operand:SI 1 "s_register_operand" "=Te")
9294         (unspec:SI [(match_dup 2)
9295                     (match_dup 3)
9296                     (subreg:SI (match_dup 4) 4)
9297                     (match_dup 5)
9298                     (match_dup 6)]
9299          VIWDUPQ_M))
9300   ]
9301   "TARGET_HAVE_MVE"
9302   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9303   [(set_attr "type" "mve_move")
9304    (set_attr "length""8")])
9305
9306 ;;
9307 ;; [vstrwq_scatter_base_wb_s vstrwq_scatter_base_wb_u]
9308 ;;
9309 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si"
9310   [(set (mem:BLK (scratch))
9311         (unspec:BLK
9312                 [(match_operand:V4SI 1 "s_register_operand" "0")
9313                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9314                  (match_operand:V4SI 3 "s_register_operand" "w")]
9315          VSTRWSBWBQ))
9316    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9317         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9318          VSTRWSBWBQ))
9319   ]
9320   "TARGET_HAVE_MVE"
9321 {
9322    rtx ops[3];
9323    ops[0] = operands[1];
9324    ops[1] = operands[2];
9325    ops[2] = operands[3];
9326    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9327    return "";
9328 }
9329   [(set_attr "length" "4")])
9330
9331 ;;
9332 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
9333 ;;
9334 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
9335  [(set (mem:BLK (scratch))
9336        (unspec:BLK
9337                 [(match_operand:V4SI 1 "s_register_operand" "0")
9338                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9339                  (match_operand:V4SI 3 "s_register_operand" "w")
9340                  (match_operand:HI 4 "vpr_register_operand")]
9341         VSTRWSBWBQ))
9342    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9343         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9344          VSTRWSBWBQ))
9345   ]
9346   "TARGET_HAVE_MVE"
9347 {
9348    rtx ops[3];
9349    ops[0] = operands[1];
9350    ops[1] = operands[2];
9351    ops[2] = operands[3];
9352    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9353    return "";
9354 }
9355   [(set_attr "length" "8")])
9356
9357 ;;
9358 ;; [vstrwq_scatter_base_wb_f]
9359 ;;
9360 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf"
9361  [(set (mem:BLK (scratch))
9362        (unspec:BLK
9363                 [(match_operand:V4SI 1 "s_register_operand" "0")
9364                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9365                  (match_operand:V4SF 3 "s_register_operand" "w")]
9366          VSTRWQSBWB_F))
9367    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9368         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9369          VSTRWQSBWB_F))
9370   ]
9371   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9372 {
9373    rtx ops[3];
9374    ops[0] = operands[1];
9375    ops[1] = operands[2];
9376    ops[2] = operands[3];
9377    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
9378    return "";
9379 }
9380   [(set_attr "length" "4")])
9381
9382 ;;
9383 ;; [vstrwq_scatter_base_wb_p_f]
9384 ;;
9385 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf"
9386  [(set (mem:BLK (scratch))
9387        (unspec:BLK
9388                 [(match_operand:V4SI 1 "s_register_operand" "0")
9389                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9390                  (match_operand:V4SF 3 "s_register_operand" "w")
9391                  (match_operand:HI 4 "vpr_register_operand")]
9392         VSTRWQSBWB_F))
9393    (set (match_operand:V4SI 0 "s_register_operand" "=w")
9394         (unspec:V4SI [(match_dup 1) (match_dup 2)]
9395          VSTRWQSBWB_F))
9396   ]
9397   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9398 {
9399    rtx ops[3];
9400    ops[0] = operands[1];
9401    ops[1] = operands[2];
9402    ops[2] = operands[3];
9403    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
9404    return "";
9405 }
9406   [(set_attr "length" "8")])
9407
9408 ;;
9409 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
9410 ;;
9411 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di"
9412   [(set (mem:BLK (scratch))
9413         (unspec:BLK
9414                 [(match_operand:V2DI 1 "s_register_operand" "0")
9415                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9416                  (match_operand:V2DI 3 "s_register_operand" "w")]
9417          VSTRDSBWBQ))
9418    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
9419         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9420          VSTRDSBWBQ))
9421   ]
9422   "TARGET_HAVE_MVE"
9423 {
9424    rtx ops[3];
9425    ops[0] = operands[1];
9426    ops[1] = operands[2];
9427    ops[2] = operands[3];
9428    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
9429    return "";
9430 }
9431   [(set_attr "length" "4")])
9432
9433 ;;
9434 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
9435 ;;
9436 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
9437   [(set (mem:BLK (scratch))
9438         (unspec:BLK
9439                 [(match_operand:V2DI 1 "s_register_operand" "0")
9440                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
9441                  (match_operand:V2DI 3 "s_register_operand" "w")
9442                  (match_operand:HI 4 "vpr_register_operand")]
9443          VSTRDSBWBQ))
9444    (set (match_operand:V2DI 0 "s_register_operand" "=w")
9445         (unspec:V2DI [(match_dup 1) (match_dup 2)]
9446          VSTRDSBWBQ))
9447   ]
9448   "TARGET_HAVE_MVE"
9449 {
9450    rtx ops[3];
9451    ops[0] = operands[1];
9452    ops[1] = operands[2];
9453    ops[2] = operands[3];
9454    output_asm_insn ("vpst;vstrdt.u64\t%q2, [%q0, %1]!",ops);
9455    return "";
9456 }
9457   [(set_attr "length" "8")])
9458
9459 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
9460   [(match_operand:V4SI 0 "s_register_operand")
9461    (match_operand:V4SI 1 "s_register_operand")
9462    (match_operand:SI 2 "mve_vldrd_immediate")
9463    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9464   "TARGET_HAVE_MVE"
9465 {
9466   rtx ignore_result = gen_reg_rtx (V4SImode);
9467   emit_insn (
9468   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
9469                                                  operands[1], operands[2]));
9470   DONE;
9471 })
9472
9473 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
9474   [(match_operand:V4SI 0 "s_register_operand")
9475    (match_operand:V4SI 1 "s_register_operand")
9476    (match_operand:SI 2 "mve_vldrd_immediate")
9477    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9478   "TARGET_HAVE_MVE"
9479 {
9480   rtx ignore_wb = gen_reg_rtx (V4SImode);
9481   emit_insn (
9482   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
9483                                                  operands[1], operands[2]));
9484   DONE;
9485 })
9486
9487 ;;
9488 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
9489 ;;
9490 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
9491   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9492         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9493                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9494                       (mem:BLK (scratch))]
9495          VLDRWGBWBQ))
9496    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9497         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9498          VLDRWGBWBQ))
9499   ]
9500   "TARGET_HAVE_MVE"
9501 {
9502    rtx ops[3];
9503    ops[0] = operands[0];
9504    ops[1] = operands[2];
9505    ops[2] = operands[3];
9506    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9507    return "";
9508 }
9509   [(set_attr "length" "4")])
9510
9511 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
9512   [(match_operand:V4SI 0 "s_register_operand")
9513    (match_operand:V4SI 1 "s_register_operand")
9514    (match_operand:SI 2 "mve_vldrd_immediate")
9515    (match_operand:HI 3 "vpr_register_operand")
9516    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9517   "TARGET_HAVE_MVE"
9518 {
9519   rtx ignore_result = gen_reg_rtx (V4SImode);
9520   emit_insn (
9521   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
9522                                                    operands[1], operands[2],
9523                                                    operands[3]));
9524   DONE;
9525 })
9526 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
9527   [(match_operand:V4SI 0 "s_register_operand")
9528    (match_operand:V4SI 1 "s_register_operand")
9529    (match_operand:SI 2 "mve_vldrd_immediate")
9530    (match_operand:HI 3 "vpr_register_operand")
9531    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
9532   "TARGET_HAVE_MVE"
9533 {
9534   rtx ignore_wb = gen_reg_rtx (V4SImode);
9535   emit_insn (
9536   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
9537                                                    operands[1], operands[2],
9538                                                    operands[3]));
9539   DONE;
9540 })
9541
9542 ;;
9543 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
9544 ;;
9545 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
9546   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
9547         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
9548                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9549                       (match_operand:HI 4 "vpr_register_operand" "Up")
9550                       (mem:BLK (scratch))]
9551          VLDRWGBWBQ))
9552    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9553         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9554          VLDRWGBWBQ))
9555   ]
9556   "TARGET_HAVE_MVE"
9557 {
9558    rtx ops[3];
9559    ops[0] = operands[0];
9560    ops[1] = operands[2];
9561    ops[2] = operands[3];
9562    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9563    return "";
9564 }
9565   [(set_attr "length" "8")])
9566
9567 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
9568   [(match_operand:V4SI 0 "s_register_operand")
9569    (match_operand:V4SI 1 "s_register_operand")
9570    (match_operand:SI 2 "mve_vldrd_immediate")
9571    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9572   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9573 {
9574   rtx ignore_result = gen_reg_rtx (V4SFmode);
9575   emit_insn (
9576   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
9577                                             operands[1], operands[2]));
9578   DONE;
9579 })
9580
9581 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
9582   [(match_operand:V4SF 0 "s_register_operand")
9583    (match_operand:V4SI 1 "s_register_operand")
9584    (match_operand:SI 2 "mve_vldrd_immediate")
9585    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9586   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9587 {
9588   rtx ignore_wb = gen_reg_rtx (V4SImode);
9589   emit_insn (
9590   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
9591                                             operands[1], operands[2]));
9592   DONE;
9593 })
9594
9595 ;;
9596 ;; [vldrwq_gather_base_wb_f]
9597 ;;
9598 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
9599   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9600         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9601                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9602                       (mem:BLK (scratch))]
9603          VLDRWQGBWB_F))
9604    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9605         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9606          VLDRWQGBWB_F))
9607   ]
9608   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9609 {
9610    rtx ops[3];
9611    ops[0] = operands[0];
9612    ops[1] = operands[2];
9613    ops[2] = operands[3];
9614    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
9615    return "";
9616 }
9617   [(set_attr "length" "4")])
9618
9619 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
9620   [(match_operand:V4SI 0 "s_register_operand")
9621    (match_operand:V4SI 1 "s_register_operand")
9622    (match_operand:SI 2 "mve_vldrd_immediate")
9623    (match_operand:HI 3 "vpr_register_operand")
9624    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9625   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9626 {
9627   rtx ignore_result = gen_reg_rtx (V4SFmode);
9628   emit_insn (
9629   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
9630                                               operands[1], operands[2],
9631                                               operands[3]));
9632   DONE;
9633 })
9634
9635 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
9636   [(match_operand:V4SF 0 "s_register_operand")
9637    (match_operand:V4SI 1 "s_register_operand")
9638    (match_operand:SI 2 "mve_vldrd_immediate")
9639    (match_operand:HI 3 "vpr_register_operand")
9640    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
9641   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9642 {
9643   rtx ignore_wb = gen_reg_rtx (V4SImode);
9644   emit_insn (
9645   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
9646                                               operands[1], operands[2],
9647                                               operands[3]));
9648   DONE;
9649 })
9650
9651 ;;
9652 ;; [vldrwq_gather_base_wb_z_f]
9653 ;;
9654 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
9655   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
9656         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
9657                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9658                       (match_operand:HI 4 "vpr_register_operand" "Up")
9659                       (mem:BLK (scratch))]
9660          VLDRWQGBWB_F))
9661    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
9662         (unspec:V4SI [(match_dup 2) (match_dup 3)]
9663          VLDRWQGBWB_F))
9664   ]
9665   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9666 {
9667    rtx ops[3];
9668    ops[0] = operands[0];
9669    ops[1] = operands[2];
9670    ops[2] = operands[3];
9671    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
9672    return "";
9673 }
9674   [(set_attr "length" "8")])
9675
9676 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
9677   [(match_operand:V2DI 0 "s_register_operand")
9678    (match_operand:V2DI 1 "s_register_operand")
9679    (match_operand:SI 2 "mve_vldrd_immediate")
9680    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9681   "TARGET_HAVE_MVE"
9682 {
9683   rtx ignore_result = gen_reg_rtx (V2DImode);
9684   emit_insn (
9685   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
9686                                                  operands[1], operands[2]));
9687   DONE;
9688 })
9689
9690 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
9691   [(match_operand:V2DI 0 "s_register_operand")
9692    (match_operand:V2DI 1 "s_register_operand")
9693    (match_operand:SI 2 "mve_vldrd_immediate")
9694    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9695   "TARGET_HAVE_MVE"
9696 {
9697   rtx ignore_wb = gen_reg_rtx (V2DImode);
9698   emit_insn (
9699   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
9700                                                  operands[1], operands[2]));
9701   DONE;
9702 })
9703
9704
9705 ;;
9706 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
9707 ;;
9708 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
9709   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9710         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9711                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9712                       (mem:BLK (scratch))]
9713          VLDRDGBWBQ))
9714    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9715         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9716          VLDRDGBWBQ))
9717   ]
9718   "TARGET_HAVE_MVE"
9719 {
9720    rtx ops[3];
9721    ops[0] = operands[0];
9722    ops[1] = operands[2];
9723    ops[2] = operands[3];
9724    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
9725    return "";
9726 }
9727   [(set_attr "length" "4")])
9728
9729 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
9730   [(match_operand:V2DI 0 "s_register_operand")
9731    (match_operand:V2DI 1 "s_register_operand")
9732    (match_operand:SI 2 "mve_vldrd_immediate")
9733    (match_operand:HI 3 "vpr_register_operand")
9734    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9735   "TARGET_HAVE_MVE"
9736 {
9737   rtx ignore_result = gen_reg_rtx (V2DImode);
9738   emit_insn (
9739   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
9740                                                    operands[1], operands[2],
9741                                                    operands[3]));
9742   DONE;
9743 })
9744
9745 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
9746   [(match_operand:V2DI 0 "s_register_operand")
9747    (match_operand:V2DI 1 "s_register_operand")
9748    (match_operand:SI 2 "mve_vldrd_immediate")
9749    (match_operand:HI 3 "vpr_register_operand")
9750    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
9751   "TARGET_HAVE_MVE"
9752 {
9753   rtx ignore_wb = gen_reg_rtx (V2DImode);
9754   emit_insn (
9755   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
9756                                                    operands[1], operands[2],
9757                                                    operands[3]));
9758   DONE;
9759 })
9760
9761 (define_insn "get_fpscr_nzcvqc"
9762  [(set (match_operand:SI 0 "register_operand" "=r")
9763    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
9764  "TARGET_HAVE_MVE"
9765  "vmrs\\t%0, FPSCR_nzcvqc"
9766   [(set_attr "type" "mve_move")])
9767
9768 (define_insn "set_fpscr_nzcvqc"
9769  [(set (reg:SI VFPCC_REGNUM)
9770    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
9771     VUNSPEC_SET_FPSCR_NZCVQC))]
9772  "TARGET_HAVE_MVE"
9773  "vmsr\\tFPSCR_nzcvqc, %0"
9774   [(set_attr "type" "mve_move")])
9775
9776 ;;
9777 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
9778 ;;
9779 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
9780   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
9781         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
9782                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
9783                       (match_operand:HI 4 "vpr_register_operand" "Up")
9784                       (mem:BLK (scratch))]
9785          VLDRDGBWBQ))
9786    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
9787         (unspec:V2DI [(match_dup 2) (match_dup 3)]
9788          VLDRDGBWBQ))
9789   ]
9790   "TARGET_HAVE_MVE"
9791 {
9792    rtx ops[3];
9793    ops[0] = operands[0];
9794    ops[1] = operands[2];
9795    ops[2] = operands[3];
9796    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
9797    return "";
9798 }
9799   [(set_attr "length" "8")])
9800 ;;
9801 ;; [vadciq_m_s, vadciq_m_u])
9802 ;;
9803 (define_insn "mve_vadciq_m_<supf>v4si"
9804   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9805         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9806                       (match_operand:V4SI 2 "s_register_operand" "w")
9807                       (match_operand:V4SI 3 "s_register_operand" "w")
9808                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9809          VADCIQ_M))
9810    (set (reg:SI VFPCC_REGNUM)
9811         (unspec:SI [(const_int 0)]
9812          VADCIQ_M))
9813   ]
9814   "TARGET_HAVE_MVE"
9815   "vpst\;vadcit.i32\t%q0, %q2, %q3"
9816   [(set_attr "type" "mve_move")
9817    (set_attr "length" "8")])
9818
9819 ;;
9820 ;; [vadciq_u, vadciq_s])
9821 ;;
9822 (define_insn "mve_vadciq_<supf>v4si"
9823   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9824         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9825                       (match_operand:V4SI 2 "s_register_operand" "w")]
9826          VADCIQ))
9827    (set (reg:SI VFPCC_REGNUM)
9828         (unspec:SI [(const_int 0)]
9829          VADCIQ))
9830   ]
9831   "TARGET_HAVE_MVE"
9832   "vadci.i32\t%q0, %q1, %q2"
9833   [(set_attr "type" "mve_move")
9834    (set_attr "length" "4")])
9835
9836 ;;
9837 ;; [vadcq_m_s, vadcq_m_u])
9838 ;;
9839 (define_insn "mve_vadcq_m_<supf>v4si"
9840   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9841         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
9842                       (match_operand:V4SI 2 "s_register_operand" "w")
9843                       (match_operand:V4SI 3 "s_register_operand" "w")
9844                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9845          VADCQ_M))
9846    (set (reg:SI VFPCC_REGNUM)
9847         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9848          VADCQ_M))
9849   ]
9850   "TARGET_HAVE_MVE"
9851   "vpst\;vadct.i32\t%q0, %q2, %q3"
9852   [(set_attr "type" "mve_move")
9853    (set_attr "length" "8")])
9854
9855 ;;
9856 ;; [vadcq_u, vadcq_s])
9857 ;;
9858 (define_insn "mve_vadcq_<supf>v4si"
9859   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9860         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9861                        (match_operand:V4SI 2 "s_register_operand" "w")]
9862          VADCQ))
9863    (set (reg:SI VFPCC_REGNUM)
9864         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9865          VADCQ))
9866   ]
9867   "TARGET_HAVE_MVE"
9868   "vadc.i32\t%q0, %q1, %q2"
9869   [(set_attr "type" "mve_move")
9870    (set_attr "length" "4")
9871    (set_attr "conds" "set")])
9872
9873 ;;
9874 ;; [vsbciq_m_u, vsbciq_m_s])
9875 ;;
9876 (define_insn "mve_vsbciq_m_<supf>v4si"
9877   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9878         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9879                       (match_operand:V4SI 2 "s_register_operand" "w")
9880                       (match_operand:V4SI 3 "s_register_operand" "w")
9881                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9882          VSBCIQ_M))
9883    (set (reg:SI VFPCC_REGNUM)
9884         (unspec:SI [(const_int 0)]
9885          VSBCIQ_M))
9886   ]
9887   "TARGET_HAVE_MVE"
9888   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
9889   [(set_attr "type" "mve_move")
9890    (set_attr "length" "8")])
9891
9892 ;;
9893 ;; [vsbciq_s, vsbciq_u])
9894 ;;
9895 (define_insn "mve_vsbciq_<supf>v4si"
9896   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9897         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9898                       (match_operand:V4SI 2 "s_register_operand" "w")]
9899          VSBCIQ))
9900    (set (reg:SI VFPCC_REGNUM)
9901         (unspec:SI [(const_int 0)]
9902          VSBCIQ))
9903   ]
9904   "TARGET_HAVE_MVE"
9905   "vsbci.i32\t%q0, %q1, %q2"
9906   [(set_attr "type" "mve_move")
9907    (set_attr "length" "4")])
9908
9909 ;;
9910 ;; [vsbcq_m_u, vsbcq_m_s])
9911 ;;
9912 (define_insn "mve_vsbcq_m_<supf>v4si"
9913   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9914         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9915                       (match_operand:V4SI 2 "s_register_operand" "w")
9916                       (match_operand:V4SI 3 "s_register_operand" "w")
9917                       (match_operand:HI 4 "vpr_register_operand" "Up")]
9918          VSBCQ_M))
9919    (set (reg:SI VFPCC_REGNUM)
9920         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9921          VSBCQ_M))
9922   ]
9923   "TARGET_HAVE_MVE"
9924   "vpst\;vsbct.i32\t%q0, %q2, %q3"
9925   [(set_attr "type" "mve_move")
9926    (set_attr "length" "8")])
9927
9928 ;;
9929 ;; [vsbcq_s, vsbcq_u])
9930 ;;
9931 (define_insn "mve_vsbcq_<supf>v4si"
9932   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
9933         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9934                       (match_operand:V4SI 2 "s_register_operand" "w")]
9935          VSBCQ))
9936    (set (reg:SI VFPCC_REGNUM)
9937         (unspec:SI [(reg:SI VFPCC_REGNUM)]
9938          VSBCQ))
9939   ]
9940   "TARGET_HAVE_MVE"
9941   "vsbc.i32\t%q0, %q1, %q2"
9942   [(set_attr "type" "mve_move")
9943    (set_attr "length" "4")])
9944
9945 ;;
9946 ;; [vst2q])
9947 ;;
9948 (define_insn "mve_vst2q<mode>"
9949   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
9950         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
9951                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9952          VST2Q))
9953   ]
9954   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9955    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9956 {
9957    rtx ops[4];
9958    int regno = REGNO (operands[1]);
9959    ops[0] = gen_rtx_REG (TImode, regno);
9960    ops[1] = gen_rtx_REG (TImode, regno + 4);
9961    rtx reg  = operands[0];
9962    while (reg && !REG_P (reg))
9963     reg = XEXP (reg, 0);
9964    gcc_assert (REG_P (reg));
9965    ops[2] = reg;
9966    ops[3] = operands[0];
9967    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
9968                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
9969    return "";
9970 }
9971   [(set_attr "length" "8")])
9972
9973 ;;
9974 ;; [vld2q])
9975 ;;
9976 (define_insn "mve_vld2q<mode>"
9977   [(set (match_operand:OI 0 "s_register_operand" "=w")
9978         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
9979                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9980          VLD2Q))
9981   ]
9982   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
9983    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
9984 {
9985    rtx ops[4];
9986    int regno = REGNO (operands[0]);
9987    ops[0] = gen_rtx_REG (TImode, regno);
9988    ops[1] = gen_rtx_REG (TImode, regno + 4);
9989    rtx reg  = operands[1];
9990    while (reg && !REG_P (reg))
9991     reg = XEXP (reg, 0);
9992    gcc_assert (REG_P (reg));
9993    ops[2] = reg;
9994    ops[3] = operands[1];
9995    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
9996                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
9997    return "";
9998 }
9999   [(set_attr "length" "8")])
10000
10001 ;;
10002 ;; [vld4q])
10003 ;;
10004 (define_insn "mve_vld4q<mode>"
10005   [(set (match_operand:XI 0 "s_register_operand" "=w")
10006         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10007                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10008          VLD4Q))
10009   ]
10010   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10011    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10012 {
10013    rtx ops[6];
10014    int regno = REGNO (operands[0]);
10015    ops[0] = gen_rtx_REG (TImode, regno);
10016    ops[1] = gen_rtx_REG (TImode, regno+4);
10017    ops[2] = gen_rtx_REG (TImode, regno+8);
10018    ops[3] = gen_rtx_REG (TImode, regno + 12);
10019    rtx reg  = operands[1];
10020    while (reg && !REG_P (reg))
10021     reg = XEXP (reg, 0);
10022    gcc_assert (REG_P (reg));
10023    ops[4] = reg;
10024    ops[5] = operands[1];
10025    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10026                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10027                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10028                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10029    return "";
10030 }
10031   [(set_attr "length" "16")])
10032 ;;
10033 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10034 ;;
10035 (define_insn "mve_vec_extract<mode><V_elem_l>"
10036  [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=r")
10037    (vec_select:<V_elem>
10038     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10039     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10040   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10041    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10042 {
10043   if (BYTES_BIG_ENDIAN)
10044     {
10045       int elt = INTVAL (operands[2]);
10046       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10047       operands[2] = GEN_INT (elt);
10048     }
10049   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
10050 }
10051  [(set_attr "type" "mve_move")])
10052
10053 (define_insn "mve_vec_extractv2didi"
10054  [(set (match_operand:DI 0 "nonimmediate_operand" "=r")
10055    (vec_select:DI
10056     (match_operand:V2DI 1 "s_register_operand" "w")
10057     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
10058   "TARGET_HAVE_MVE"
10059 {
10060   int elt = INTVAL (operands[2]);
10061   if (BYTES_BIG_ENDIAN)
10062     elt = 1 - elt;
10063
10064   if (elt == 0)
10065    return "vmov\t%Q0, %R0, %e1";
10066   else
10067    return "vmov\t%Q0, %R0, %f1";
10068 }
10069  [(set_attr "type" "mve_move")])
10070
10071 (define_insn "*mve_vec_extract_sext_internal<mode>"
10072  [(set (match_operand:SI 0 "s_register_operand" "=r")
10073    (sign_extend:SI
10074     (vec_select:<V_elem>
10075      (match_operand:MVE_2 1 "s_register_operand" "w")
10076      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10077   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10078    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10079 {
10080   if (BYTES_BIG_ENDIAN)
10081     {
10082       int elt = INTVAL (operands[2]);
10083       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10084       operands[2] = GEN_INT (elt);
10085     }
10086   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
10087 }
10088  [(set_attr "type" "mve_move")])
10089
10090 (define_insn "*mve_vec_extract_zext_internal<mode>"
10091  [(set (match_operand:SI 0 "s_register_operand" "=r")
10092    (zero_extend:SI
10093     (vec_select:<V_elem>
10094      (match_operand:MVE_2 1 "s_register_operand" "w")
10095      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
10096   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10097    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10098 {
10099   if (BYTES_BIG_ENDIAN)
10100     {
10101       int elt = INTVAL (operands[2]);
10102       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10103       operands[2] = GEN_INT (elt);
10104     }
10105   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
10106 }
10107  [(set_attr "type" "mve_move")])
10108
10109 ;;
10110 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
10111 ;;
10112 (define_insn "mve_vec_set<mode>_internal"
10113  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
10114        (vec_merge:VQ2
10115         (vec_duplicate:VQ2
10116           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
10117         (match_operand:VQ2 3 "s_register_operand" "0")
10118         (match_operand:SI 2 "immediate_operand" "i")))]
10119   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10120    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10121 {
10122   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10123   if (BYTES_BIG_ENDIAN)
10124     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
10125   operands[2] = GEN_INT (elt);
10126
10127   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
10128 }
10129  [(set_attr "type" "mve_move")])
10130
10131 (define_insn "mve_vec_setv2di_internal"
10132  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
10133        (vec_merge:V2DI
10134         (vec_duplicate:V2DI
10135           (match_operand:DI 1 "nonimmediate_operand" "r"))
10136         (match_operand:V2DI 3 "s_register_operand" "0")
10137         (match_operand:SI 2 "immediate_operand" "i")))]
10138  "TARGET_HAVE_MVE"
10139 {
10140   int elt = ffs ((int) INTVAL (operands[2])) - 1;
10141   if (BYTES_BIG_ENDIAN)
10142     elt = 1 - elt;
10143
10144   if (elt == 0)
10145    return "vmov\t%e0, %Q1, %R1";
10146   else
10147    return "vmov\t%f0, %J1, %K1";
10148 }
10149  [(set_attr "type" "mve_move")])
10150
10151 ;;
10152 ;; [uqrshll_di]
10153 ;;
10154 (define_insn "mve_uqrshll_sat<supf>_di"
10155   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10156         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10157                     (match_operand:SI 2 "register_operand" "r")]
10158          UQRSHLLQ))]
10159   "TARGET_HAVE_MVE"
10160   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
10161   [(set_attr "predicable" "yes")])
10162
10163 ;;
10164 ;; [sqrshrl_di]
10165 ;;
10166 (define_insn "mve_sqrshrl_sat<supf>_di"
10167   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10168         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10169                     (match_operand:SI 2 "register_operand" "r")]
10170          SQRSHRLQ))]
10171   "TARGET_HAVE_MVE"
10172   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
10173   [(set_attr "predicable" "yes")])
10174
10175 ;;
10176 ;; [uqrshl_si]
10177 ;;
10178 (define_insn "mve_uqrshl_si"
10179   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10180         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10181                     (match_operand:SI 2 "register_operand" "r")]
10182          UQRSHL))]
10183   "TARGET_HAVE_MVE"
10184   "uqrshl%?\\t%1, %2"
10185   [(set_attr "predicable" "yes")])
10186
10187 ;;
10188 ;; [sqrshr_si]
10189 ;;
10190 (define_insn "mve_sqrshr_si"
10191   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10192         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10193                     (match_operand:SI 2 "register_operand" "r")]
10194          SQRSHR))]
10195   "TARGET_HAVE_MVE"
10196   "sqrshr%?\\t%1, %2"
10197   [(set_attr "predicable" "yes")])
10198
10199 ;;
10200 ;; [uqshll_di]
10201 ;;
10202 (define_insn "mve_uqshll_di"
10203   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10204         (us_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10205                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10206   "TARGET_HAVE_MVE"
10207   "uqshll%?\\t%Q1, %R1, %2"
10208   [(set_attr "predicable" "yes")])
10209
10210 ;;
10211 ;; [urshrl_di]
10212 ;;
10213 (define_insn "mve_urshrl_di"
10214   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10215         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10216                     (match_operand:SI 2 "immediate_operand" "Pg")]
10217          URSHRL))]
10218   "TARGET_HAVE_MVE"
10219   "urshrl%?\\t%Q1, %R1, %2"
10220   [(set_attr "predicable" "yes")])
10221
10222 ;;
10223 ;; [uqshl_si]
10224 ;;
10225 (define_insn "mve_uqshl_si"
10226   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10227         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "0")
10228                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10229   "TARGET_HAVE_MVE"
10230   "uqshl%?\\t%1, %2"
10231   [(set_attr "predicable" "yes")])
10232
10233 ;;
10234 ;; [urshr_si]
10235 ;;
10236 (define_insn "mve_urshr_si"
10237   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10238         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "0")
10239                     (match_operand:SI 2 "immediate_operand" "Pg")]
10240          URSHR))]
10241   "TARGET_HAVE_MVE"
10242   "urshr%?\\t%1, %2"
10243   [(set_attr "predicable" "yes")])
10244
10245 ;;
10246 ;; [sqshl_si]
10247 ;;
10248 (define_insn "mve_sqshl_si"
10249   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10250         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "0")
10251                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10252   "TARGET_HAVE_MVE"
10253   "sqshl%?\\t%1, %2"
10254   [(set_attr "predicable" "yes")])
10255
10256 ;;
10257 ;; [srshr_si]
10258 ;;
10259 (define_insn "mve_srshr_si"
10260   [(set (match_operand:SI 0 "arm_general_register_operand" "=r")
10261         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "0")
10262                     (match_operand:SI 2 "immediate_operand" "Pg")]
10263          SRSHR))]
10264   "TARGET_HAVE_MVE"
10265   "srshr%?\\t%1, %2"
10266   [(set_attr "predicable" "yes")])
10267
10268 ;;
10269 ;; [srshrl_di]
10270 ;;
10271 (define_insn "mve_srshrl_di"
10272   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10273         (unspec:DI [(match_operand:DI 1 "arm_low_register_operand" "0")
10274                     (match_operand:SI 2 "immediate_operand" "Pg")]
10275          SRSHRL))]
10276   "TARGET_HAVE_MVE"
10277   "srshrl%?\\t%Q1, %R1, %2"
10278   [(set_attr "predicable" "yes")])
10279
10280 ;;
10281 ;; [sqshll_di]
10282 ;;
10283 (define_insn "mve_sqshll_di"
10284   [(set (match_operand:DI 0 "arm_low_register_operand" "=l")
10285         (ss_ashift:DI (match_operand:DI 1 "arm_low_register_operand" "0")
10286                       (match_operand:SI 2 "immediate_operand" "Pg")))]
10287   "TARGET_HAVE_MVE"
10288   "sqshll%?\\t%Q1, %R1, %2"
10289   [(set_attr "predicable" "yes")])
10290
10291 ;;
10292 ;; [vshlcq_m_u vshlcq_m_s]
10293 ;;
10294 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
10295  [(match_operand:MVE_2 0 "s_register_operand")
10296   (match_operand:MVE_2 1 "s_register_operand")
10297   (match_operand:SI 2 "s_register_operand")
10298   (match_operand:SI 3 "mve_imm_32")
10299   (match_operand:HI 4 "vpr_register_operand")
10300   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10301  "TARGET_HAVE_MVE"
10302 {
10303   rtx ignore_wb = gen_reg_rtx (SImode);
10304   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
10305                                             operands[2], operands[3],
10306                                             operands[4]));
10307   DONE;
10308 })
10309
10310 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
10311  [(match_operand:SI 0 "s_register_operand")
10312   (match_operand:MVE_2 1 "s_register_operand")
10313   (match_operand:SI 2 "s_register_operand")
10314   (match_operand:SI 3 "mve_imm_32")
10315   (match_operand:HI 4 "vpr_register_operand")
10316   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
10317  "TARGET_HAVE_MVE"
10318 {
10319   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10320   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
10321                                             operands[1], operands[2],
10322                                             operands[3], operands[4]));
10323   DONE;
10324 })
10325
10326 (define_insn "mve_vshlcq_m_<supf><mode>"
10327  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10328        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10329                       (match_operand:SI 3 "s_register_operand" "1")
10330                       (match_operand:SI 4 "mve_imm_32" "Rf")
10331                       (match_operand:HI 5 "vpr_register_operand" "Up")]
10332         VSHLCQ_M))
10333   (set (match_operand:SI  1 "s_register_operand" "=r")
10334        (unspec:SI [(match_dup 2)
10335                    (match_dup 3)
10336                    (match_dup 4)
10337                    (match_dup 5)]
10338         VSHLCQ_M))
10339  ]
10340  "TARGET_HAVE_MVE"
10341  "vpst\;vshlct\t%q0, %1, %4"
10342  [(set_attr "type" "mve_move")
10343   (set_attr "length" "8")])
10344
10345 ;; CDE instructions on MVE registers.
10346
10347 (define_insn "arm_vcx1qv16qi"
10348   [(set (match_operand:V16QI 0 "register_operand" "=t")
10349         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10350                            (match_operand:SI 2 "const_int_mve_cde1_operand" "i")]
10351          UNSPEC_VCDE))]
10352   "TARGET_CDE && TARGET_HAVE_MVE"
10353   "vcx1\\tp%c1, %q0, #%c2"
10354   [(set_attr "type" "coproc")]
10355 )
10356
10357 (define_insn "arm_vcx1qav16qi"
10358   [(set (match_operand:V16QI 0 "register_operand" "=t")
10359         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10360                             (match_operand:V16QI 2 "register_operand" "0")
10361                             (match_operand:SI 3 "const_int_mve_cde1_operand" "i")]
10362          UNSPEC_VCDEA))]
10363   "TARGET_CDE && TARGET_HAVE_MVE"
10364   "vcx1a\\tp%c1, %q0, #%c3"
10365   [(set_attr "type" "coproc")]
10366 )
10367
10368 (define_insn "arm_vcx2qv16qi"
10369   [(set (match_operand:V16QI 0 "register_operand" "=t")
10370         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10371                           (match_operand:V16QI 2 "register_operand" "t")
10372                           (match_operand:SI 3 "const_int_mve_cde2_operand" "i")]
10373          UNSPEC_VCDE))]
10374   "TARGET_CDE && TARGET_HAVE_MVE"
10375   "vcx2\\tp%c1, %q0, %q2, #%c3"
10376   [(set_attr "type" "coproc")]
10377 )
10378
10379 (define_insn "arm_vcx2qav16qi"
10380   [(set (match_operand:V16QI 0 "register_operand" "=t")
10381         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10382                           (match_operand:V16QI 2 "register_operand" "0")
10383                           (match_operand:V16QI 3 "register_operand" "t")
10384                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")]
10385          UNSPEC_VCDEA))]
10386   "TARGET_CDE && TARGET_HAVE_MVE"
10387   "vcx2a\\tp%c1, %q0, %q3, #%c4"
10388   [(set_attr "type" "coproc")]
10389 )
10390
10391 (define_insn "arm_vcx3qv16qi"
10392   [(set (match_operand:V16QI 0 "register_operand" "=t")
10393         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10394                           (match_operand:V16QI 2 "register_operand" "t")
10395                           (match_operand:V16QI 3 "register_operand" "t")
10396                           (match_operand:SI 4 "const_int_mve_cde3_operand" "i")]
10397          UNSPEC_VCDE))]
10398   "TARGET_CDE && TARGET_HAVE_MVE"
10399   "vcx3\\tp%c1, %q0, %q2, %q3, #%c4"
10400   [(set_attr "type" "coproc")]
10401 )
10402
10403 (define_insn "arm_vcx3qav16qi"
10404   [(set (match_operand:V16QI 0 "register_operand" "=t")
10405         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10406                           (match_operand:V16QI 2 "register_operand" "0")
10407                           (match_operand:V16QI 3 "register_operand" "t")
10408                           (match_operand:V16QI 4 "register_operand" "t")
10409                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")]
10410          UNSPEC_VCDEA))]
10411   "TARGET_CDE && TARGET_HAVE_MVE"
10412   "vcx3a\\tp%c1, %q0, %q3, %q4, #%c5"
10413   [(set_attr "type" "coproc")]
10414 )
10415
10416 (define_insn "arm_vcx1q<a>_p_v16qi"
10417   [(set (match_operand:V16QI 0 "register_operand" "=t")
10418         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10419                            (match_operand:V16QI 2 "register_operand" "0")
10420                            (match_operand:SI 3 "const_int_mve_cde1_operand" "i")
10421                            (match_operand:HI 4 "vpr_register_operand" "Up")]
10422          CDE_VCX))]
10423   "TARGET_CDE && TARGET_HAVE_MVE"
10424   "vpst\;vcx1<a>t\\tp%c1, %q0, #%c3"
10425   [(set_attr "type" "coproc")
10426    (set_attr "length" "8")]
10427 )
10428
10429 (define_insn "arm_vcx2q<a>_p_v16qi"
10430   [(set (match_operand:V16QI 0 "register_operand" "=t")
10431         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10432                           (match_operand:V16QI 2 "register_operand" "0")
10433                           (match_operand:V16QI 3 "register_operand" "t")
10434                           (match_operand:SI 4 "const_int_mve_cde2_operand" "i")
10435                           (match_operand:HI 5 "vpr_register_operand" "Up")]
10436          CDE_VCX))]
10437   "TARGET_CDE && TARGET_HAVE_MVE"
10438   "vpst\;vcx2<a>t\\tp%c1, %q0, %q3, #%c4"
10439   [(set_attr "type" "coproc")
10440    (set_attr "length" "8")]
10441 )
10442
10443 (define_insn "arm_vcx3q<a>_p_v16qi"
10444   [(set (match_operand:V16QI 0 "register_operand" "=t")
10445         (unspec:V16QI [(match_operand:SI 1 "const_int_coproc_operand" "i")
10446                           (match_operand:V16QI 2 "register_operand" "0")
10447                           (match_operand:V16QI 3 "register_operand" "t")
10448                           (match_operand:V16QI 4 "register_operand" "t")
10449                           (match_operand:SI 5 "const_int_mve_cde3_operand" "i")
10450                           (match_operand:HI 6 "vpr_register_operand" "Up")]
10451          CDE_VCX))]
10452   "TARGET_CDE && TARGET_HAVE_MVE"
10453   "vpst\;vcx3<a>t\\tp%c1, %q0, %q3, %q4, #%c5"
10454   [(set_attr "type" "coproc")
10455    (set_attr "length" "8")]
10456 )
10457
10458 (define_insn "*movmisalign<mode>_mve_store"
10459   [(set (match_operand:MVE_VLD_ST 0 "neon_permissive_struct_operand"         "=Ux")
10460         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "s_register_operand" " w")]
10461          UNSPEC_MISALIGNED_ACCESS))]
10462   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10463     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10464    && !BYTES_BIG_ENDIAN && unaligned_access"
10465   "vstr<V_sz_elem1>.<V_sz_elem>\t%q1, %E0"
10466   [(set_attr "type" "mve_store")]
10467 )
10468
10469
10470 (define_insn "*movmisalign<mode>_mve_load"
10471   [(set (match_operand:MVE_VLD_ST 0 "s_register_operand"                                 "=w")
10472         (unspec:MVE_VLD_ST [(match_operand:MVE_VLD_ST 1 "neon_permissive_struct_operand" " Ux")]
10473          UNSPEC_MISALIGNED_ACCESS))]
10474   "((TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10475     || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode)))
10476    && !BYTES_BIG_ENDIAN && unaligned_access"
10477   "vldr<V_sz_elem1>.<V_sz_elem>\t%q0, %E1"
10478   [(set_attr "type" "mve_load")]
10479 )